1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2017, Microsoft Corporation.
4 * Copyright (C) 2018, LG Electronics.
5 *
6 * Author(s): Long Li <longli@microsoft.com>,
7 * Hyunchul Lee <hyc.lee@gmail.com>
8 */
9
10 #define SUBMOD_NAME "smb_direct"
11
12 #include <linux/kthread.h>
13 #include <linux/list.h>
14 #include <linux/mempool.h>
15 #include <linux/highmem.h>
16 #include <linux/scatterlist.h>
17 #include <rdma/ib_verbs.h>
18 #include <rdma/rdma_cm.h>
19 #include <rdma/rw.h>
20
21 #include "glob.h"
22 #include "connection.h"
23 #include "smb_common.h"
24 #include "../common/smb2status.h"
25 #include "transport_rdma.h"
26
27 #define SMB_DIRECT_PORT_IWARP 5445
28 #define SMB_DIRECT_PORT_INFINIBAND 445
29
30 #define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100)
31
32 /* SMB_DIRECT negotiation timeout in seconds */
33 #define SMB_DIRECT_NEGOTIATE_TIMEOUT 120
34
35 #define SMB_DIRECT_MAX_SEND_SGES 6
36 #define SMB_DIRECT_MAX_RECV_SGES 1
37
38 /*
39 * Default maximum number of RDMA read/write outstanding on this connection
40 * This value is possibly decreased during QP creation on hardware limit
41 */
42 #define SMB_DIRECT_CM_INITIATOR_DEPTH 8
43
44 /* Maximum number of retries on data transfer operations */
45 #define SMB_DIRECT_CM_RETRY 6
46 /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
47 #define SMB_DIRECT_CM_RNR_RETRY 0
48
49 /*
50 * User configurable initial values per SMB_DIRECT transport connection
51 * as defined in [MS-SMBD] 3.1.1.1
52 * Those may change after a SMB_DIRECT negotiation
53 */
54
55 /* Set 445 port to SMB Direct port by default */
56 static int smb_direct_port = SMB_DIRECT_PORT_INFINIBAND;
57
58 /* The local peer's maximum number of credits to grant to the peer */
59 static int smb_direct_receive_credit_max = 255;
60
61 /* The remote peer's credit request of local peer */
62 static int smb_direct_send_credit_target = 255;
63
64 /* The maximum single message size can be sent to remote peer */
65 static int smb_direct_max_send_size = 1364;
66
67 /* The maximum fragmented upper-layer payload receive size supported */
68 static int smb_direct_max_fragmented_recv_size = 1024 * 1024;
69
70 /* The maximum single-message size which can be received */
71 static int smb_direct_max_receive_size = 1364;
72
73 static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE;
74
75 static LIST_HEAD(smb_direct_device_list);
76 static DEFINE_RWLOCK(smb_direct_device_lock);
77
78 struct smb_direct_device {
79 struct ib_device *ib_dev;
80 struct list_head list;
81 };
82
83 static struct smb_direct_listener {
84 struct rdma_cm_id *cm_id;
85 } smb_direct_listener;
86
87 static struct workqueue_struct *smb_direct_wq;
88
89 enum smb_direct_status {
90 SMB_DIRECT_CS_NEW = 0,
91 SMB_DIRECT_CS_CONNECTED,
92 SMB_DIRECT_CS_DISCONNECTING,
93 SMB_DIRECT_CS_DISCONNECTED,
94 };
95
96 struct smb_direct_transport {
97 struct ksmbd_transport transport;
98
99 enum smb_direct_status status;
100 bool full_packet_received;
101 wait_queue_head_t wait_status;
102
103 struct rdma_cm_id *cm_id;
104 struct ib_cq *send_cq;
105 struct ib_cq *recv_cq;
106 struct ib_pd *pd;
107 struct ib_qp *qp;
108
109 int max_send_size;
110 int max_recv_size;
111 int max_fragmented_send_size;
112 int max_fragmented_recv_size;
113 int max_rdma_rw_size;
114
115 spinlock_t reassembly_queue_lock;
116 struct list_head reassembly_queue;
117 int reassembly_data_length;
118 int reassembly_queue_length;
119 int first_entry_offset;
120 wait_queue_head_t wait_reassembly_queue;
121
122 spinlock_t receive_credit_lock;
123 int recv_credits;
124 int count_avail_recvmsg;
125 int recv_credit_max;
126 int recv_credit_target;
127
128 spinlock_t recvmsg_queue_lock;
129 struct list_head recvmsg_queue;
130
131 int send_credit_target;
132 atomic_t send_credits;
133 spinlock_t lock_new_recv_credits;
134 int new_recv_credits;
135 int max_rw_credits;
136 int pages_per_rw_credit;
137 atomic_t rw_credits;
138
139 wait_queue_head_t wait_send_credits;
140 wait_queue_head_t wait_rw_credits;
141
142 mempool_t *sendmsg_mempool;
143 struct kmem_cache *sendmsg_cache;
144 mempool_t *recvmsg_mempool;
145 struct kmem_cache *recvmsg_cache;
146
147 wait_queue_head_t wait_send_pending;
148 atomic_t send_pending;
149
150 struct work_struct post_recv_credits_work;
151 struct work_struct send_immediate_work;
152 struct work_struct disconnect_work;
153
154 bool negotiation_requested;
155 };
156
157 #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport))
158 #define SMBD_TRANS(t) ((struct smb_direct_transport *)container_of(t, \
159 struct smb_direct_transport, transport))
160 enum {
161 SMB_DIRECT_MSG_NEGOTIATE_REQ = 0,
162 SMB_DIRECT_MSG_DATA_TRANSFER
163 };
164
165 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops;
166
167 struct smb_direct_send_ctx {
168 struct list_head msg_list;
169 int wr_cnt;
170 bool need_invalidate_rkey;
171 unsigned int remote_key;
172 };
173
174 struct smb_direct_sendmsg {
175 struct smb_direct_transport *transport;
176 struct ib_send_wr wr;
177 struct list_head list;
178 int num_sge;
179 struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES];
180 struct ib_cqe cqe;
181 u8 packet[];
182 };
183
184 struct smb_direct_recvmsg {
185 struct smb_direct_transport *transport;
186 struct list_head list;
187 int type;
188 struct ib_sge sge;
189 struct ib_cqe cqe;
190 bool first_segment;
191 u8 packet[];
192 };
193
194 struct smb_direct_rdma_rw_msg {
195 struct smb_direct_transport *t;
196 struct ib_cqe cqe;
197 int status;
198 struct completion *completion;
199 struct list_head list;
200 struct rdma_rw_ctx rw_ctx;
201 struct sg_table sgt;
202 struct scatterlist sg_list[];
203 };
204
init_smbd_max_io_size(unsigned int sz)205 void init_smbd_max_io_size(unsigned int sz)
206 {
207 sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE);
208 smb_direct_max_read_write_size = sz;
209 }
210
get_smbd_max_read_write_size(void)211 unsigned int get_smbd_max_read_write_size(void)
212 {
213 return smb_direct_max_read_write_size;
214 }
215
get_buf_page_count(void * buf,int size)216 static inline int get_buf_page_count(void *buf, int size)
217 {
218 return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
219 (uintptr_t)buf / PAGE_SIZE;
220 }
221
222 static void smb_direct_destroy_pools(struct smb_direct_transport *transport);
223 static void smb_direct_post_recv_credits(struct work_struct *work);
224 static int smb_direct_post_send_data(struct smb_direct_transport *t,
225 struct smb_direct_send_ctx *send_ctx,
226 struct kvec *iov, int niov,
227 int remaining_data_length);
228
229 static inline struct smb_direct_transport *
smb_trans_direct_transfort(struct ksmbd_transport * t)230 smb_trans_direct_transfort(struct ksmbd_transport *t)
231 {
232 return container_of(t, struct smb_direct_transport, transport);
233 }
234
235 static inline void
smb_direct_recvmsg_payload(struct smb_direct_recvmsg * recvmsg)236 *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg)
237 {
238 return (void *)recvmsg->packet;
239 }
240
is_receive_credit_post_required(int receive_credits,int avail_recvmsg_count)241 static inline bool is_receive_credit_post_required(int receive_credits,
242 int avail_recvmsg_count)
243 {
244 return receive_credits <= (smb_direct_receive_credit_max >> 3) &&
245 avail_recvmsg_count >= (receive_credits >> 2);
246 }
247
248 static struct
get_free_recvmsg(struct smb_direct_transport * t)249 smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
250 {
251 struct smb_direct_recvmsg *recvmsg = NULL;
252
253 spin_lock(&t->recvmsg_queue_lock);
254 if (!list_empty(&t->recvmsg_queue)) {
255 recvmsg = list_first_entry(&t->recvmsg_queue,
256 struct smb_direct_recvmsg,
257 list);
258 list_del(&recvmsg->list);
259 }
260 spin_unlock(&t->recvmsg_queue_lock);
261 return recvmsg;
262 }
263
put_recvmsg(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg)264 static void put_recvmsg(struct smb_direct_transport *t,
265 struct smb_direct_recvmsg *recvmsg)
266 {
267 if (likely(recvmsg->sge.length != 0)) {
268 ib_dma_unmap_single(t->cm_id->device,
269 recvmsg->sge.addr,
270 recvmsg->sge.length,
271 DMA_FROM_DEVICE);
272 recvmsg->sge.length = 0;
273 }
274
275 spin_lock(&t->recvmsg_queue_lock);
276 list_add(&recvmsg->list, &t->recvmsg_queue);
277 spin_unlock(&t->recvmsg_queue_lock);
278 }
279
enqueue_reassembly(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg,int data_length)280 static void enqueue_reassembly(struct smb_direct_transport *t,
281 struct smb_direct_recvmsg *recvmsg,
282 int data_length)
283 {
284 spin_lock(&t->reassembly_queue_lock);
285 list_add_tail(&recvmsg->list, &t->reassembly_queue);
286 t->reassembly_queue_length++;
287 /*
288 * Make sure reassembly_data_length is updated after list and
289 * reassembly_queue_length are updated. On the dequeue side
290 * reassembly_data_length is checked without a lock to determine
291 * if reassembly_queue_length and list is up to date
292 */
293 virt_wmb();
294 t->reassembly_data_length += data_length;
295 spin_unlock(&t->reassembly_queue_lock);
296 }
297
get_first_reassembly(struct smb_direct_transport * t)298 static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t)
299 {
300 if (!list_empty(&t->reassembly_queue))
301 return list_first_entry(&t->reassembly_queue,
302 struct smb_direct_recvmsg, list);
303 else
304 return NULL;
305 }
306
smb_direct_disconnect_rdma_work(struct work_struct * work)307 static void smb_direct_disconnect_rdma_work(struct work_struct *work)
308 {
309 struct smb_direct_transport *t =
310 container_of(work, struct smb_direct_transport,
311 disconnect_work);
312
313 if (t->status == SMB_DIRECT_CS_CONNECTED) {
314 t->status = SMB_DIRECT_CS_DISCONNECTING;
315 rdma_disconnect(t->cm_id);
316 }
317 }
318
319 static void
smb_direct_disconnect_rdma_connection(struct smb_direct_transport * t)320 smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t)
321 {
322 if (t->status == SMB_DIRECT_CS_CONNECTED)
323 queue_work(smb_direct_wq, &t->disconnect_work);
324 }
325
smb_direct_send_immediate_work(struct work_struct * work)326 static void smb_direct_send_immediate_work(struct work_struct *work)
327 {
328 struct smb_direct_transport *t = container_of(work,
329 struct smb_direct_transport, send_immediate_work);
330
331 if (t->status != SMB_DIRECT_CS_CONNECTED)
332 return;
333
334 smb_direct_post_send_data(t, NULL, NULL, 0, 0);
335 }
336
alloc_transport(struct rdma_cm_id * cm_id)337 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
338 {
339 struct smb_direct_transport *t;
340 struct ksmbd_conn *conn;
341
342 t = kzalloc(sizeof(*t), KSMBD_DEFAULT_GFP);
343 if (!t)
344 return NULL;
345
346 t->cm_id = cm_id;
347 cm_id->context = t;
348
349 t->status = SMB_DIRECT_CS_NEW;
350 init_waitqueue_head(&t->wait_status);
351
352 spin_lock_init(&t->reassembly_queue_lock);
353 INIT_LIST_HEAD(&t->reassembly_queue);
354 t->reassembly_data_length = 0;
355 t->reassembly_queue_length = 0;
356 init_waitqueue_head(&t->wait_reassembly_queue);
357 init_waitqueue_head(&t->wait_send_credits);
358 init_waitqueue_head(&t->wait_rw_credits);
359
360 spin_lock_init(&t->receive_credit_lock);
361 spin_lock_init(&t->recvmsg_queue_lock);
362 INIT_LIST_HEAD(&t->recvmsg_queue);
363
364 init_waitqueue_head(&t->wait_send_pending);
365 atomic_set(&t->send_pending, 0);
366
367 spin_lock_init(&t->lock_new_recv_credits);
368
369 INIT_WORK(&t->post_recv_credits_work,
370 smb_direct_post_recv_credits);
371 INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work);
372 INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work);
373
374 conn = ksmbd_conn_alloc();
375 if (!conn)
376 goto err;
377 conn->transport = KSMBD_TRANS(t);
378 KSMBD_TRANS(t)->conn = conn;
379 KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
380 return t;
381 err:
382 kfree(t);
383 return NULL;
384 }
385
smb_direct_free_transport(struct ksmbd_transport * kt)386 static void smb_direct_free_transport(struct ksmbd_transport *kt)
387 {
388 kfree(SMBD_TRANS(kt));
389 }
390
free_transport(struct smb_direct_transport * t)391 static void free_transport(struct smb_direct_transport *t)
392 {
393 struct smb_direct_recvmsg *recvmsg;
394
395 wake_up_interruptible(&t->wait_send_credits);
396
397 ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n");
398 wait_event(t->wait_send_pending,
399 atomic_read(&t->send_pending) == 0);
400
401 disable_work_sync(&t->disconnect_work);
402 disable_work_sync(&t->post_recv_credits_work);
403 disable_work_sync(&t->send_immediate_work);
404
405 if (t->qp) {
406 ib_drain_qp(t->qp);
407 ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs);
408 t->qp = NULL;
409 rdma_destroy_qp(t->cm_id);
410 }
411
412 ksmbd_debug(RDMA, "drain the reassembly queue\n");
413 do {
414 spin_lock(&t->reassembly_queue_lock);
415 recvmsg = get_first_reassembly(t);
416 if (recvmsg) {
417 list_del(&recvmsg->list);
418 spin_unlock(&t->reassembly_queue_lock);
419 put_recvmsg(t, recvmsg);
420 } else {
421 spin_unlock(&t->reassembly_queue_lock);
422 }
423 } while (recvmsg);
424 t->reassembly_data_length = 0;
425
426 if (t->send_cq)
427 ib_free_cq(t->send_cq);
428 if (t->recv_cq)
429 ib_free_cq(t->recv_cq);
430 if (t->pd)
431 ib_dealloc_pd(t->pd);
432 if (t->cm_id)
433 rdma_destroy_id(t->cm_id);
434
435 smb_direct_destroy_pools(t);
436 ksmbd_conn_free(KSMBD_TRANS(t)->conn);
437 }
438
439 static struct smb_direct_sendmsg
smb_direct_alloc_sendmsg(struct smb_direct_transport * t)440 *smb_direct_alloc_sendmsg(struct smb_direct_transport *t)
441 {
442 struct smb_direct_sendmsg *msg;
443
444 msg = mempool_alloc(t->sendmsg_mempool, KSMBD_DEFAULT_GFP);
445 if (!msg)
446 return ERR_PTR(-ENOMEM);
447 msg->transport = t;
448 INIT_LIST_HEAD(&msg->list);
449 msg->num_sge = 0;
450 return msg;
451 }
452
smb_direct_free_sendmsg(struct smb_direct_transport * t,struct smb_direct_sendmsg * msg)453 static void smb_direct_free_sendmsg(struct smb_direct_transport *t,
454 struct smb_direct_sendmsg *msg)
455 {
456 int i;
457
458 if (msg->num_sge > 0) {
459 ib_dma_unmap_single(t->cm_id->device,
460 msg->sge[0].addr, msg->sge[0].length,
461 DMA_TO_DEVICE);
462 for (i = 1; i < msg->num_sge; i++)
463 ib_dma_unmap_page(t->cm_id->device,
464 msg->sge[i].addr, msg->sge[i].length,
465 DMA_TO_DEVICE);
466 }
467 mempool_free(msg, t->sendmsg_mempool);
468 }
469
smb_direct_check_recvmsg(struct smb_direct_recvmsg * recvmsg)470 static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg)
471 {
472 switch (recvmsg->type) {
473 case SMB_DIRECT_MSG_DATA_TRANSFER: {
474 struct smb_direct_data_transfer *req =
475 (struct smb_direct_data_transfer *)recvmsg->packet;
476 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
477 + le32_to_cpu(req->data_offset));
478 ksmbd_debug(RDMA,
479 "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
480 le16_to_cpu(req->credits_granted),
481 le16_to_cpu(req->credits_requested),
482 req->data_length, req->remaining_data_length,
483 hdr->ProtocolId, hdr->Command);
484 break;
485 }
486 case SMB_DIRECT_MSG_NEGOTIATE_REQ: {
487 struct smb_direct_negotiate_req *req =
488 (struct smb_direct_negotiate_req *)recvmsg->packet;
489 ksmbd_debug(RDMA,
490 "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
491 le16_to_cpu(req->min_version),
492 le16_to_cpu(req->max_version),
493 le16_to_cpu(req->credits_requested),
494 le32_to_cpu(req->preferred_send_size),
495 le32_to_cpu(req->max_receive_size),
496 le32_to_cpu(req->max_fragmented_size));
497 if (le16_to_cpu(req->min_version) > 0x0100 ||
498 le16_to_cpu(req->max_version) < 0x0100)
499 return -EOPNOTSUPP;
500 if (le16_to_cpu(req->credits_requested) <= 0 ||
501 le32_to_cpu(req->max_receive_size) <= 128 ||
502 le32_to_cpu(req->max_fragmented_size) <=
503 128 * 1024)
504 return -ECONNABORTED;
505
506 break;
507 }
508 default:
509 return -EINVAL;
510 }
511 return 0;
512 }
513
recv_done(struct ib_cq * cq,struct ib_wc * wc)514 static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
515 {
516 struct smb_direct_recvmsg *recvmsg;
517 struct smb_direct_transport *t;
518
519 recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe);
520 t = recvmsg->transport;
521
522 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
523 put_recvmsg(t, recvmsg);
524 if (wc->status != IB_WC_WR_FLUSH_ERR) {
525 pr_err("Recv error. status='%s (%d)' opcode=%d\n",
526 ib_wc_status_msg(wc->status), wc->status,
527 wc->opcode);
528 smb_direct_disconnect_rdma_connection(t);
529 }
530 return;
531 }
532
533 ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
534 ib_wc_status_msg(wc->status), wc->status,
535 wc->opcode);
536
537 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
538 recvmsg->sge.length, DMA_FROM_DEVICE);
539
540 switch (recvmsg->type) {
541 case SMB_DIRECT_MSG_NEGOTIATE_REQ:
542 if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) {
543 put_recvmsg(t, recvmsg);
544 smb_direct_disconnect_rdma_connection(t);
545 return;
546 }
547 t->negotiation_requested = true;
548 t->full_packet_received = true;
549 t->status = SMB_DIRECT_CS_CONNECTED;
550 enqueue_reassembly(t, recvmsg, 0);
551 wake_up_interruptible(&t->wait_status);
552 return;
553 case SMB_DIRECT_MSG_DATA_TRANSFER: {
554 struct smb_direct_data_transfer *data_transfer =
555 (struct smb_direct_data_transfer *)recvmsg->packet;
556 u32 remaining_data_length, data_offset, data_length;
557 int avail_recvmsg_count, receive_credits;
558
559 if (wc->byte_len <
560 offsetof(struct smb_direct_data_transfer, padding)) {
561 put_recvmsg(t, recvmsg);
562 smb_direct_disconnect_rdma_connection(t);
563 return;
564 }
565
566 remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length);
567 data_length = le32_to_cpu(data_transfer->data_length);
568 data_offset = le32_to_cpu(data_transfer->data_offset);
569 if (wc->byte_len < data_offset ||
570 wc->byte_len < (u64)data_offset + data_length) {
571 put_recvmsg(t, recvmsg);
572 smb_direct_disconnect_rdma_connection(t);
573 return;
574 }
575 if (remaining_data_length > t->max_fragmented_recv_size ||
576 data_length > t->max_fragmented_recv_size ||
577 (u64)remaining_data_length + (u64)data_length >
578 (u64)t->max_fragmented_recv_size) {
579 put_recvmsg(t, recvmsg);
580 smb_direct_disconnect_rdma_connection(t);
581 return;
582 }
583
584 if (data_length) {
585 if (t->full_packet_received)
586 recvmsg->first_segment = true;
587
588 if (le32_to_cpu(data_transfer->remaining_data_length))
589 t->full_packet_received = false;
590 else
591 t->full_packet_received = true;
592
593 spin_lock(&t->receive_credit_lock);
594 receive_credits = --(t->recv_credits);
595 avail_recvmsg_count = t->count_avail_recvmsg;
596 spin_unlock(&t->receive_credit_lock);
597 } else {
598 spin_lock(&t->receive_credit_lock);
599 receive_credits = --(t->recv_credits);
600 avail_recvmsg_count = ++(t->count_avail_recvmsg);
601 spin_unlock(&t->receive_credit_lock);
602 }
603
604 t->recv_credit_target =
605 le16_to_cpu(data_transfer->credits_requested);
606 atomic_add(le16_to_cpu(data_transfer->credits_granted),
607 &t->send_credits);
608
609 if (le16_to_cpu(data_transfer->flags) &
610 SMB_DIRECT_RESPONSE_REQUESTED)
611 queue_work(smb_direct_wq, &t->send_immediate_work);
612
613 if (atomic_read(&t->send_credits) > 0)
614 wake_up_interruptible(&t->wait_send_credits);
615
616 if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count))
617 queue_work(smb_direct_wq, &t->post_recv_credits_work);
618
619 if (data_length) {
620 enqueue_reassembly(t, recvmsg, (int)data_length);
621 wake_up_interruptible(&t->wait_reassembly_queue);
622 } else
623 put_recvmsg(t, recvmsg);
624
625 return;
626 }
627 }
628
629 /*
630 * This is an internal error!
631 */
632 WARN_ON_ONCE(recvmsg->type != SMB_DIRECT_MSG_DATA_TRANSFER);
633 put_recvmsg(t, recvmsg);
634 smb_direct_disconnect_rdma_connection(t);
635 }
636
smb_direct_post_recv(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg)637 static int smb_direct_post_recv(struct smb_direct_transport *t,
638 struct smb_direct_recvmsg *recvmsg)
639 {
640 struct ib_recv_wr wr;
641 int ret;
642
643 recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device,
644 recvmsg->packet, t->max_recv_size,
645 DMA_FROM_DEVICE);
646 ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr);
647 if (ret)
648 return ret;
649 recvmsg->sge.length = t->max_recv_size;
650 recvmsg->sge.lkey = t->pd->local_dma_lkey;
651 recvmsg->cqe.done = recv_done;
652
653 wr.wr_cqe = &recvmsg->cqe;
654 wr.next = NULL;
655 wr.sg_list = &recvmsg->sge;
656 wr.num_sge = 1;
657
658 ret = ib_post_recv(t->qp, &wr, NULL);
659 if (ret) {
660 pr_err("Can't post recv: %d\n", ret);
661 ib_dma_unmap_single(t->cm_id->device,
662 recvmsg->sge.addr, recvmsg->sge.length,
663 DMA_FROM_DEVICE);
664 recvmsg->sge.length = 0;
665 smb_direct_disconnect_rdma_connection(t);
666 return ret;
667 }
668 return ret;
669 }
670
smb_direct_read(struct ksmbd_transport * t,char * buf,unsigned int size,int unused)671 static int smb_direct_read(struct ksmbd_transport *t, char *buf,
672 unsigned int size, int unused)
673 {
674 struct smb_direct_recvmsg *recvmsg;
675 struct smb_direct_data_transfer *data_transfer;
676 int to_copy, to_read, data_read, offset;
677 u32 data_length, remaining_data_length, data_offset;
678 int rc;
679 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
680
681 again:
682 if (st->status != SMB_DIRECT_CS_CONNECTED) {
683 pr_err("disconnected\n");
684 return -ENOTCONN;
685 }
686
687 /*
688 * No need to hold the reassembly queue lock all the time as we are
689 * the only one reading from the front of the queue. The transport
690 * may add more entries to the back of the queue at the same time
691 */
692 if (st->reassembly_data_length >= size) {
693 int queue_length;
694 int queue_removed = 0;
695
696 /*
697 * Need to make sure reassembly_data_length is read before
698 * reading reassembly_queue_length and calling
699 * get_first_reassembly. This call is lock free
700 * as we never read at the end of the queue which are being
701 * updated in SOFTIRQ as more data is received
702 */
703 virt_rmb();
704 queue_length = st->reassembly_queue_length;
705 data_read = 0;
706 to_read = size;
707 offset = st->first_entry_offset;
708 while (data_read < size) {
709 recvmsg = get_first_reassembly(st);
710 data_transfer = smb_direct_recvmsg_payload(recvmsg);
711 data_length = le32_to_cpu(data_transfer->data_length);
712 remaining_data_length =
713 le32_to_cpu(data_transfer->remaining_data_length);
714 data_offset = le32_to_cpu(data_transfer->data_offset);
715
716 /*
717 * The upper layer expects RFC1002 length at the
718 * beginning of the payload. Return it to indicate
719 * the total length of the packet. This minimize the
720 * change to upper layer packet processing logic. This
721 * will be eventually remove when an intermediate
722 * transport layer is added
723 */
724 if (recvmsg->first_segment && size == 4) {
725 unsigned int rfc1002_len =
726 data_length + remaining_data_length;
727 *((__be32 *)buf) = cpu_to_be32(rfc1002_len);
728 data_read = 4;
729 recvmsg->first_segment = false;
730 ksmbd_debug(RDMA,
731 "returning rfc1002 length %d\n",
732 rfc1002_len);
733 goto read_rfc1002_done;
734 }
735
736 to_copy = min_t(int, data_length - offset, to_read);
737 memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
738 to_copy);
739
740 /* move on to the next buffer? */
741 if (to_copy == data_length - offset) {
742 queue_length--;
743 /*
744 * No need to lock if we are not at the
745 * end of the queue
746 */
747 if (queue_length) {
748 list_del(&recvmsg->list);
749 } else {
750 spin_lock_irq(&st->reassembly_queue_lock);
751 list_del(&recvmsg->list);
752 spin_unlock_irq(&st->reassembly_queue_lock);
753 }
754 queue_removed++;
755 put_recvmsg(st, recvmsg);
756 offset = 0;
757 } else {
758 offset += to_copy;
759 }
760
761 to_read -= to_copy;
762 data_read += to_copy;
763 }
764
765 spin_lock_irq(&st->reassembly_queue_lock);
766 st->reassembly_data_length -= data_read;
767 st->reassembly_queue_length -= queue_removed;
768 spin_unlock_irq(&st->reassembly_queue_lock);
769
770 spin_lock(&st->receive_credit_lock);
771 st->count_avail_recvmsg += queue_removed;
772 if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) {
773 spin_unlock(&st->receive_credit_lock);
774 queue_work(smb_direct_wq, &st->post_recv_credits_work);
775 } else {
776 spin_unlock(&st->receive_credit_lock);
777 }
778
779 st->first_entry_offset = offset;
780 ksmbd_debug(RDMA,
781 "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
782 data_read, st->reassembly_data_length,
783 st->first_entry_offset);
784 read_rfc1002_done:
785 return data_read;
786 }
787
788 ksmbd_debug(RDMA, "wait_event on more data\n");
789 rc = wait_event_interruptible(st->wait_reassembly_queue,
790 st->reassembly_data_length >= size ||
791 st->status != SMB_DIRECT_CS_CONNECTED);
792 if (rc)
793 return -EINTR;
794
795 goto again;
796 }
797
smb_direct_post_recv_credits(struct work_struct * work)798 static void smb_direct_post_recv_credits(struct work_struct *work)
799 {
800 struct smb_direct_transport *t = container_of(work,
801 struct smb_direct_transport, post_recv_credits_work);
802 struct smb_direct_recvmsg *recvmsg;
803 int receive_credits, credits = 0;
804 int ret;
805
806 spin_lock(&t->receive_credit_lock);
807 receive_credits = t->recv_credits;
808 spin_unlock(&t->receive_credit_lock);
809
810 if (receive_credits < t->recv_credit_target) {
811 while (true) {
812 recvmsg = get_free_recvmsg(t);
813 if (!recvmsg)
814 break;
815
816 recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER;
817 recvmsg->first_segment = false;
818
819 ret = smb_direct_post_recv(t, recvmsg);
820 if (ret) {
821 pr_err("Can't post recv: %d\n", ret);
822 put_recvmsg(t, recvmsg);
823 break;
824 }
825 credits++;
826 }
827 }
828
829 spin_lock(&t->receive_credit_lock);
830 t->recv_credits += credits;
831 t->count_avail_recvmsg -= credits;
832 spin_unlock(&t->receive_credit_lock);
833
834 spin_lock(&t->lock_new_recv_credits);
835 t->new_recv_credits += credits;
836 spin_unlock(&t->lock_new_recv_credits);
837
838 if (credits)
839 queue_work(smb_direct_wq, &t->send_immediate_work);
840 }
841
send_done(struct ib_cq * cq,struct ib_wc * wc)842 static void send_done(struct ib_cq *cq, struct ib_wc *wc)
843 {
844 struct smb_direct_sendmsg *sendmsg, *sibling;
845 struct smb_direct_transport *t;
846 struct list_head *pos, *prev, *end;
847
848 sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe);
849 t = sendmsg->transport;
850
851 ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
852 ib_wc_status_msg(wc->status), wc->status,
853 wc->opcode);
854
855 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
856 pr_err("Send error. status='%s (%d)', opcode=%d\n",
857 ib_wc_status_msg(wc->status), wc->status,
858 wc->opcode);
859 smb_direct_disconnect_rdma_connection(t);
860 }
861
862 if (atomic_dec_and_test(&t->send_pending))
863 wake_up(&t->wait_send_pending);
864
865 /* iterate and free the list of messages in reverse. the list's head
866 * is invalid.
867 */
868 for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next;
869 prev != end; pos = prev, prev = prev->prev) {
870 sibling = container_of(pos, struct smb_direct_sendmsg, list);
871 smb_direct_free_sendmsg(t, sibling);
872 }
873
874 sibling = container_of(pos, struct smb_direct_sendmsg, list);
875 smb_direct_free_sendmsg(t, sibling);
876 }
877
manage_credits_prior_sending(struct smb_direct_transport * t)878 static int manage_credits_prior_sending(struct smb_direct_transport *t)
879 {
880 int new_credits;
881
882 spin_lock(&t->lock_new_recv_credits);
883 new_credits = t->new_recv_credits;
884 t->new_recv_credits = 0;
885 spin_unlock(&t->lock_new_recv_credits);
886
887 return new_credits;
888 }
889
smb_direct_post_send(struct smb_direct_transport * t,struct ib_send_wr * wr)890 static int smb_direct_post_send(struct smb_direct_transport *t,
891 struct ib_send_wr *wr)
892 {
893 int ret;
894
895 atomic_inc(&t->send_pending);
896 ret = ib_post_send(t->qp, wr, NULL);
897 if (ret) {
898 pr_err("failed to post send: %d\n", ret);
899 if (atomic_dec_and_test(&t->send_pending))
900 wake_up(&t->wait_send_pending);
901 smb_direct_disconnect_rdma_connection(t);
902 }
903 return ret;
904 }
905
smb_direct_send_ctx_init(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,bool need_invalidate_rkey,unsigned int remote_key)906 static void smb_direct_send_ctx_init(struct smb_direct_transport *t,
907 struct smb_direct_send_ctx *send_ctx,
908 bool need_invalidate_rkey,
909 unsigned int remote_key)
910 {
911 INIT_LIST_HEAD(&send_ctx->msg_list);
912 send_ctx->wr_cnt = 0;
913 send_ctx->need_invalidate_rkey = need_invalidate_rkey;
914 send_ctx->remote_key = remote_key;
915 }
916
smb_direct_flush_send_list(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,bool is_last)917 static int smb_direct_flush_send_list(struct smb_direct_transport *t,
918 struct smb_direct_send_ctx *send_ctx,
919 bool is_last)
920 {
921 struct smb_direct_sendmsg *first, *last;
922 int ret;
923
924 if (list_empty(&send_ctx->msg_list))
925 return 0;
926
927 first = list_first_entry(&send_ctx->msg_list,
928 struct smb_direct_sendmsg,
929 list);
930 last = list_last_entry(&send_ctx->msg_list,
931 struct smb_direct_sendmsg,
932 list);
933
934 last->wr.send_flags = IB_SEND_SIGNALED;
935 last->wr.wr_cqe = &last->cqe;
936 if (is_last && send_ctx->need_invalidate_rkey) {
937 last->wr.opcode = IB_WR_SEND_WITH_INV;
938 last->wr.ex.invalidate_rkey = send_ctx->remote_key;
939 }
940
941 ret = smb_direct_post_send(t, &first->wr);
942 if (!ret) {
943 smb_direct_send_ctx_init(t, send_ctx,
944 send_ctx->need_invalidate_rkey,
945 send_ctx->remote_key);
946 } else {
947 atomic_add(send_ctx->wr_cnt, &t->send_credits);
948 wake_up(&t->wait_send_credits);
949 list_for_each_entry_safe(first, last, &send_ctx->msg_list,
950 list) {
951 smb_direct_free_sendmsg(t, first);
952 }
953 }
954 return ret;
955 }
956
wait_for_credits(struct smb_direct_transport * t,wait_queue_head_t * waitq,atomic_t * total_credits,int needed)957 static int wait_for_credits(struct smb_direct_transport *t,
958 wait_queue_head_t *waitq, atomic_t *total_credits,
959 int needed)
960 {
961 int ret;
962
963 do {
964 if (atomic_sub_return(needed, total_credits) >= 0)
965 return 0;
966
967 atomic_add(needed, total_credits);
968 ret = wait_event_interruptible(*waitq,
969 atomic_read(total_credits) >= needed ||
970 t->status != SMB_DIRECT_CS_CONNECTED);
971
972 if (t->status != SMB_DIRECT_CS_CONNECTED)
973 return -ENOTCONN;
974 else if (ret < 0)
975 return ret;
976 } while (true);
977 }
978
wait_for_send_credits(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx)979 static int wait_for_send_credits(struct smb_direct_transport *t,
980 struct smb_direct_send_ctx *send_ctx)
981 {
982 int ret;
983
984 if (send_ctx &&
985 (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) {
986 ret = smb_direct_flush_send_list(t, send_ctx, false);
987 if (ret)
988 return ret;
989 }
990
991 return wait_for_credits(t, &t->wait_send_credits, &t->send_credits, 1);
992 }
993
wait_for_rw_credits(struct smb_direct_transport * t,int credits)994 static int wait_for_rw_credits(struct smb_direct_transport *t, int credits)
995 {
996 return wait_for_credits(t, &t->wait_rw_credits, &t->rw_credits, credits);
997 }
998
calc_rw_credits(struct smb_direct_transport * t,char * buf,unsigned int len)999 static int calc_rw_credits(struct smb_direct_transport *t,
1000 char *buf, unsigned int len)
1001 {
1002 return DIV_ROUND_UP(get_buf_page_count(buf, len),
1003 t->pages_per_rw_credit);
1004 }
1005
smb_direct_create_header(struct smb_direct_transport * t,int size,int remaining_data_length,struct smb_direct_sendmsg ** sendmsg_out)1006 static int smb_direct_create_header(struct smb_direct_transport *t,
1007 int size, int remaining_data_length,
1008 struct smb_direct_sendmsg **sendmsg_out)
1009 {
1010 struct smb_direct_sendmsg *sendmsg;
1011 struct smb_direct_data_transfer *packet;
1012 int header_length;
1013 int ret;
1014
1015 sendmsg = smb_direct_alloc_sendmsg(t);
1016 if (IS_ERR(sendmsg))
1017 return PTR_ERR(sendmsg);
1018
1019 /* Fill in the packet header */
1020 packet = (struct smb_direct_data_transfer *)sendmsg->packet;
1021 packet->credits_requested = cpu_to_le16(t->send_credit_target);
1022 packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1023
1024 packet->flags = 0;
1025 packet->reserved = 0;
1026 if (!size)
1027 packet->data_offset = 0;
1028 else
1029 packet->data_offset = cpu_to_le32(24);
1030 packet->data_length = cpu_to_le32(size);
1031 packet->remaining_data_length = cpu_to_le32(remaining_data_length);
1032 packet->padding = 0;
1033
1034 ksmbd_debug(RDMA,
1035 "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
1036 le16_to_cpu(packet->credits_requested),
1037 le16_to_cpu(packet->credits_granted),
1038 le32_to_cpu(packet->data_offset),
1039 le32_to_cpu(packet->data_length),
1040 le32_to_cpu(packet->remaining_data_length));
1041
1042 /* Map the packet to DMA */
1043 header_length = sizeof(struct smb_direct_data_transfer);
1044 /* If this is a packet without payload, don't send padding */
1045 if (!size)
1046 header_length =
1047 offsetof(struct smb_direct_data_transfer, padding);
1048
1049 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1050 (void *)packet,
1051 header_length,
1052 DMA_TO_DEVICE);
1053 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1054 if (ret) {
1055 smb_direct_free_sendmsg(t, sendmsg);
1056 return ret;
1057 }
1058
1059 sendmsg->num_sge = 1;
1060 sendmsg->sge[0].length = header_length;
1061 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1062
1063 *sendmsg_out = sendmsg;
1064 return 0;
1065 }
1066
get_sg_list(void * buf,int size,struct scatterlist * sg_list,int nentries)1067 static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
1068 {
1069 bool high = is_vmalloc_addr(buf);
1070 struct page *page;
1071 int offset, len;
1072 int i = 0;
1073
1074 if (size <= 0 || nentries < get_buf_page_count(buf, size))
1075 return -EINVAL;
1076
1077 offset = offset_in_page(buf);
1078 buf -= offset;
1079 while (size > 0) {
1080 len = min_t(int, PAGE_SIZE - offset, size);
1081 if (high)
1082 page = vmalloc_to_page(buf);
1083 else
1084 page = kmap_to_page(buf);
1085
1086 if (!sg_list)
1087 return -EINVAL;
1088 sg_set_page(sg_list, page, len, offset);
1089 sg_list = sg_next(sg_list);
1090
1091 buf += PAGE_SIZE;
1092 size -= len;
1093 offset = 0;
1094 i++;
1095 }
1096 return i;
1097 }
1098
get_mapped_sg_list(struct ib_device * device,void * buf,int size,struct scatterlist * sg_list,int nentries,enum dma_data_direction dir)1099 static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
1100 struct scatterlist *sg_list, int nentries,
1101 enum dma_data_direction dir)
1102 {
1103 int npages;
1104
1105 npages = get_sg_list(buf, size, sg_list, nentries);
1106 if (npages < 0)
1107 return -EINVAL;
1108 return ib_dma_map_sg(device, sg_list, npages, dir);
1109 }
1110
post_sendmsg(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,struct smb_direct_sendmsg * msg)1111 static int post_sendmsg(struct smb_direct_transport *t,
1112 struct smb_direct_send_ctx *send_ctx,
1113 struct smb_direct_sendmsg *msg)
1114 {
1115 int i;
1116
1117 for (i = 0; i < msg->num_sge; i++)
1118 ib_dma_sync_single_for_device(t->cm_id->device,
1119 msg->sge[i].addr, msg->sge[i].length,
1120 DMA_TO_DEVICE);
1121
1122 msg->cqe.done = send_done;
1123 msg->wr.opcode = IB_WR_SEND;
1124 msg->wr.sg_list = &msg->sge[0];
1125 msg->wr.num_sge = msg->num_sge;
1126 msg->wr.next = NULL;
1127
1128 if (send_ctx) {
1129 msg->wr.wr_cqe = NULL;
1130 msg->wr.send_flags = 0;
1131 if (!list_empty(&send_ctx->msg_list)) {
1132 struct smb_direct_sendmsg *last;
1133
1134 last = list_last_entry(&send_ctx->msg_list,
1135 struct smb_direct_sendmsg,
1136 list);
1137 last->wr.next = &msg->wr;
1138 }
1139 list_add_tail(&msg->list, &send_ctx->msg_list);
1140 send_ctx->wr_cnt++;
1141 return 0;
1142 }
1143
1144 msg->wr.wr_cqe = &msg->cqe;
1145 msg->wr.send_flags = IB_SEND_SIGNALED;
1146 return smb_direct_post_send(t, &msg->wr);
1147 }
1148
smb_direct_post_send_data(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,struct kvec * iov,int niov,int remaining_data_length)1149 static int smb_direct_post_send_data(struct smb_direct_transport *t,
1150 struct smb_direct_send_ctx *send_ctx,
1151 struct kvec *iov, int niov,
1152 int remaining_data_length)
1153 {
1154 int i, j, ret;
1155 struct smb_direct_sendmsg *msg;
1156 int data_length;
1157 struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1];
1158
1159 ret = wait_for_send_credits(t, send_ctx);
1160 if (ret)
1161 return ret;
1162
1163 data_length = 0;
1164 for (i = 0; i < niov; i++)
1165 data_length += iov[i].iov_len;
1166
1167 ret = smb_direct_create_header(t, data_length, remaining_data_length,
1168 &msg);
1169 if (ret) {
1170 atomic_inc(&t->send_credits);
1171 return ret;
1172 }
1173
1174 for (i = 0; i < niov; i++) {
1175 struct ib_sge *sge;
1176 int sg_cnt;
1177
1178 sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1);
1179 sg_cnt = get_mapped_sg_list(t->cm_id->device,
1180 iov[i].iov_base, iov[i].iov_len,
1181 sg, SMB_DIRECT_MAX_SEND_SGES - 1,
1182 DMA_TO_DEVICE);
1183 if (sg_cnt <= 0) {
1184 pr_err("failed to map buffer\n");
1185 ret = -ENOMEM;
1186 goto err;
1187 } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) {
1188 pr_err("buffer not fitted into sges\n");
1189 ret = -E2BIG;
1190 ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt,
1191 DMA_TO_DEVICE);
1192 goto err;
1193 }
1194
1195 for (j = 0; j < sg_cnt; j++) {
1196 sge = &msg->sge[msg->num_sge];
1197 sge->addr = sg_dma_address(&sg[j]);
1198 sge->length = sg_dma_len(&sg[j]);
1199 sge->lkey = t->pd->local_dma_lkey;
1200 msg->num_sge++;
1201 }
1202 }
1203
1204 ret = post_sendmsg(t, send_ctx, msg);
1205 if (ret)
1206 goto err;
1207 return 0;
1208 err:
1209 smb_direct_free_sendmsg(t, msg);
1210 atomic_inc(&t->send_credits);
1211 return ret;
1212 }
1213
smb_direct_writev(struct ksmbd_transport * t,struct kvec * iov,int niovs,int buflen,bool need_invalidate,unsigned int remote_key)1214 static int smb_direct_writev(struct ksmbd_transport *t,
1215 struct kvec *iov, int niovs, int buflen,
1216 bool need_invalidate, unsigned int remote_key)
1217 {
1218 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1219 int remaining_data_length;
1220 int start, i, j;
1221 int max_iov_size = st->max_send_size -
1222 sizeof(struct smb_direct_data_transfer);
1223 int ret;
1224 struct kvec vec;
1225 struct smb_direct_send_ctx send_ctx;
1226
1227 if (st->status != SMB_DIRECT_CS_CONNECTED)
1228 return -ENOTCONN;
1229
1230 //FIXME: skip RFC1002 header..
1231 buflen -= 4;
1232
1233 remaining_data_length = buflen;
1234 ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
1235
1236 smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
1237 start = i = 1;
1238 buflen = 0;
1239 while (true) {
1240 buflen += iov[i].iov_len;
1241 if (buflen > max_iov_size) {
1242 if (i > start) {
1243 remaining_data_length -=
1244 (buflen - iov[i].iov_len);
1245 ret = smb_direct_post_send_data(st, &send_ctx,
1246 &iov[start], i - start,
1247 remaining_data_length);
1248 if (ret)
1249 goto done;
1250 } else {
1251 /* iov[start] is too big, break it */
1252 int nvec = (buflen + max_iov_size - 1) /
1253 max_iov_size;
1254
1255 for (j = 0; j < nvec; j++) {
1256 vec.iov_base =
1257 (char *)iov[start].iov_base +
1258 j * max_iov_size;
1259 vec.iov_len =
1260 min_t(int, max_iov_size,
1261 buflen - max_iov_size * j);
1262 remaining_data_length -= vec.iov_len;
1263 ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
1264 remaining_data_length);
1265 if (ret)
1266 goto done;
1267 }
1268 i++;
1269 if (i == niovs)
1270 break;
1271 }
1272 start = i;
1273 buflen = 0;
1274 } else {
1275 i++;
1276 if (i == niovs) {
1277 /* send out all remaining vecs */
1278 remaining_data_length -= buflen;
1279 ret = smb_direct_post_send_data(st, &send_ctx,
1280 &iov[start], i - start,
1281 remaining_data_length);
1282 if (ret)
1283 goto done;
1284 break;
1285 }
1286 }
1287 }
1288
1289 done:
1290 ret = smb_direct_flush_send_list(st, &send_ctx, true);
1291
1292 /*
1293 * As an optimization, we don't wait for individual I/O to finish
1294 * before sending the next one.
1295 * Send them all and wait for pending send count to get to 0
1296 * that means all the I/Os have been out and we are good to return
1297 */
1298
1299 wait_event(st->wait_send_pending,
1300 atomic_read(&st->send_pending) == 0);
1301 return ret;
1302 }
1303
smb_direct_free_rdma_rw_msg(struct smb_direct_transport * t,struct smb_direct_rdma_rw_msg * msg,enum dma_data_direction dir)1304 static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t,
1305 struct smb_direct_rdma_rw_msg *msg,
1306 enum dma_data_direction dir)
1307 {
1308 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
1309 msg->sgt.sgl, msg->sgt.nents, dir);
1310 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1311 kfree(msg);
1312 }
1313
read_write_done(struct ib_cq * cq,struct ib_wc * wc,enum dma_data_direction dir)1314 static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
1315 enum dma_data_direction dir)
1316 {
1317 struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,
1318 struct smb_direct_rdma_rw_msg, cqe);
1319 struct smb_direct_transport *t = msg->t;
1320
1321 if (wc->status != IB_WC_SUCCESS) {
1322 msg->status = -EIO;
1323 pr_err("read/write error. opcode = %d, status = %s(%d)\n",
1324 wc->opcode, ib_wc_status_msg(wc->status), wc->status);
1325 if (wc->status != IB_WC_WR_FLUSH_ERR)
1326 smb_direct_disconnect_rdma_connection(t);
1327 }
1328
1329 complete(msg->completion);
1330 }
1331
read_done(struct ib_cq * cq,struct ib_wc * wc)1332 static void read_done(struct ib_cq *cq, struct ib_wc *wc)
1333 {
1334 read_write_done(cq, wc, DMA_FROM_DEVICE);
1335 }
1336
write_done(struct ib_cq * cq,struct ib_wc * wc)1337 static void write_done(struct ib_cq *cq, struct ib_wc *wc)
1338 {
1339 read_write_done(cq, wc, DMA_TO_DEVICE);
1340 }
1341
smb_direct_rdma_xmit(struct smb_direct_transport * t,void * buf,int buf_len,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len,bool is_read)1342 static int smb_direct_rdma_xmit(struct smb_direct_transport *t,
1343 void *buf, int buf_len,
1344 struct smb2_buffer_desc_v1 *desc,
1345 unsigned int desc_len,
1346 bool is_read)
1347 {
1348 struct smb_direct_rdma_rw_msg *msg, *next_msg;
1349 int i, ret;
1350 DECLARE_COMPLETION_ONSTACK(completion);
1351 struct ib_send_wr *first_wr;
1352 LIST_HEAD(msg_list);
1353 char *desc_buf;
1354 int credits_needed;
1355 unsigned int desc_buf_len, desc_num = 0;
1356
1357 if (t->status != SMB_DIRECT_CS_CONNECTED)
1358 return -ENOTCONN;
1359
1360 if (buf_len > t->max_rdma_rw_size)
1361 return -EINVAL;
1362
1363 /* calculate needed credits */
1364 credits_needed = 0;
1365 desc_buf = buf;
1366 for (i = 0; i < desc_len / sizeof(*desc); i++) {
1367 if (!buf_len)
1368 break;
1369
1370 desc_buf_len = le32_to_cpu(desc[i].length);
1371 if (!desc_buf_len)
1372 return -EINVAL;
1373
1374 if (desc_buf_len > buf_len) {
1375 desc_buf_len = buf_len;
1376 desc[i].length = cpu_to_le32(desc_buf_len);
1377 buf_len = 0;
1378 }
1379
1380 credits_needed += calc_rw_credits(t, desc_buf, desc_buf_len);
1381 desc_buf += desc_buf_len;
1382 buf_len -= desc_buf_len;
1383 desc_num++;
1384 }
1385
1386 ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n",
1387 is_read ? "read" : "write", buf_len, credits_needed);
1388
1389 ret = wait_for_rw_credits(t, credits_needed);
1390 if (ret < 0)
1391 return ret;
1392
1393 /* build rdma_rw_ctx for each descriptor */
1394 desc_buf = buf;
1395 for (i = 0; i < desc_num; i++) {
1396 msg = kzalloc(struct_size(msg, sg_list, SG_CHUNK_SIZE),
1397 KSMBD_DEFAULT_GFP);
1398 if (!msg) {
1399 ret = -ENOMEM;
1400 goto out;
1401 }
1402
1403 desc_buf_len = le32_to_cpu(desc[i].length);
1404
1405 msg->t = t;
1406 msg->cqe.done = is_read ? read_done : write_done;
1407 msg->completion = &completion;
1408
1409 msg->sgt.sgl = &msg->sg_list[0];
1410 ret = sg_alloc_table_chained(&msg->sgt,
1411 get_buf_page_count(desc_buf, desc_buf_len),
1412 msg->sg_list, SG_CHUNK_SIZE);
1413 if (ret) {
1414 kfree(msg);
1415 ret = -ENOMEM;
1416 goto out;
1417 }
1418
1419 ret = get_sg_list(desc_buf, desc_buf_len,
1420 msg->sgt.sgl, msg->sgt.orig_nents);
1421 if (ret < 0) {
1422 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1423 kfree(msg);
1424 goto out;
1425 }
1426
1427 ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,
1428 msg->sgt.sgl,
1429 get_buf_page_count(desc_buf, desc_buf_len),
1430 0,
1431 le64_to_cpu(desc[i].offset),
1432 le32_to_cpu(desc[i].token),
1433 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1434 if (ret < 0) {
1435 pr_err("failed to init rdma_rw_ctx: %d\n", ret);
1436 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1437 kfree(msg);
1438 goto out;
1439 }
1440
1441 list_add_tail(&msg->list, &msg_list);
1442 desc_buf += desc_buf_len;
1443 }
1444
1445 /* concatenate work requests of rdma_rw_ctxs */
1446 first_wr = NULL;
1447 list_for_each_entry_reverse(msg, &msg_list, list) {
1448 first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,
1449 &msg->cqe, first_wr);
1450 }
1451
1452 ret = ib_post_send(t->qp, first_wr, NULL);
1453 if (ret) {
1454 pr_err("failed to post send wr for RDMA R/W: %d\n", ret);
1455 goto out;
1456 }
1457
1458 msg = list_last_entry(&msg_list, struct smb_direct_rdma_rw_msg, list);
1459 wait_for_completion(&completion);
1460 ret = msg->status;
1461 out:
1462 list_for_each_entry_safe(msg, next_msg, &msg_list, list) {
1463 list_del(&msg->list);
1464 smb_direct_free_rdma_rw_msg(t, msg,
1465 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1466 }
1467 atomic_add(credits_needed, &t->rw_credits);
1468 wake_up(&t->wait_rw_credits);
1469 return ret;
1470 }
1471
smb_direct_rdma_write(struct ksmbd_transport * t,void * buf,unsigned int buflen,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len)1472 static int smb_direct_rdma_write(struct ksmbd_transport *t,
1473 void *buf, unsigned int buflen,
1474 struct smb2_buffer_desc_v1 *desc,
1475 unsigned int desc_len)
1476 {
1477 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1478 desc, desc_len, false);
1479 }
1480
smb_direct_rdma_read(struct ksmbd_transport * t,void * buf,unsigned int buflen,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len)1481 static int smb_direct_rdma_read(struct ksmbd_transport *t,
1482 void *buf, unsigned int buflen,
1483 struct smb2_buffer_desc_v1 *desc,
1484 unsigned int desc_len)
1485 {
1486 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1487 desc, desc_len, true);
1488 }
1489
smb_direct_disconnect(struct ksmbd_transport * t)1490 static void smb_direct_disconnect(struct ksmbd_transport *t)
1491 {
1492 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1493
1494 ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id);
1495
1496 smb_direct_disconnect_rdma_work(&st->disconnect_work);
1497 wait_event_interruptible(st->wait_status,
1498 st->status == SMB_DIRECT_CS_DISCONNECTED);
1499 free_transport(st);
1500 }
1501
smb_direct_shutdown(struct ksmbd_transport * t)1502 static void smb_direct_shutdown(struct ksmbd_transport *t)
1503 {
1504 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1505
1506 ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", st->cm_id);
1507
1508 smb_direct_disconnect_rdma_work(&st->disconnect_work);
1509 }
1510
smb_direct_cm_handler(struct rdma_cm_id * cm_id,struct rdma_cm_event * event)1511 static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
1512 struct rdma_cm_event *event)
1513 {
1514 struct smb_direct_transport *t = cm_id->context;
1515
1516 ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
1517 cm_id, rdma_event_msg(event->event), event->event);
1518
1519 switch (event->event) {
1520 case RDMA_CM_EVENT_ESTABLISHED: {
1521 t->status = SMB_DIRECT_CS_CONNECTED;
1522 wake_up_interruptible(&t->wait_status);
1523 break;
1524 }
1525 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1526 case RDMA_CM_EVENT_DISCONNECTED: {
1527 ib_drain_qp(t->qp);
1528
1529 t->status = SMB_DIRECT_CS_DISCONNECTED;
1530 wake_up_interruptible(&t->wait_status);
1531 wake_up_interruptible(&t->wait_reassembly_queue);
1532 wake_up(&t->wait_send_credits);
1533 break;
1534 }
1535 case RDMA_CM_EVENT_CONNECT_ERROR: {
1536 t->status = SMB_DIRECT_CS_DISCONNECTED;
1537 wake_up_interruptible(&t->wait_status);
1538 break;
1539 }
1540 default:
1541 pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
1542 cm_id, rdma_event_msg(event->event),
1543 event->event);
1544 break;
1545 }
1546 return 0;
1547 }
1548
smb_direct_qpair_handler(struct ib_event * event,void * context)1549 static void smb_direct_qpair_handler(struct ib_event *event, void *context)
1550 {
1551 struct smb_direct_transport *t = context;
1552
1553 ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
1554 t->cm_id, ib_event_msg(event->event), event->event);
1555
1556 switch (event->event) {
1557 case IB_EVENT_CQ_ERR:
1558 case IB_EVENT_QP_FATAL:
1559 smb_direct_disconnect_rdma_connection(t);
1560 break;
1561 default:
1562 break;
1563 }
1564 }
1565
smb_direct_send_negotiate_response(struct smb_direct_transport * t,int failed)1566 static int smb_direct_send_negotiate_response(struct smb_direct_transport *t,
1567 int failed)
1568 {
1569 struct smb_direct_sendmsg *sendmsg;
1570 struct smb_direct_negotiate_resp *resp;
1571 int ret;
1572
1573 sendmsg = smb_direct_alloc_sendmsg(t);
1574 if (IS_ERR(sendmsg))
1575 return -ENOMEM;
1576
1577 resp = (struct smb_direct_negotiate_resp *)sendmsg->packet;
1578 if (failed) {
1579 memset(resp, 0, sizeof(*resp));
1580 resp->min_version = cpu_to_le16(0x0100);
1581 resp->max_version = cpu_to_le16(0x0100);
1582 resp->status = STATUS_NOT_SUPPORTED;
1583 } else {
1584 resp->status = STATUS_SUCCESS;
1585 resp->min_version = SMB_DIRECT_VERSION_LE;
1586 resp->max_version = SMB_DIRECT_VERSION_LE;
1587 resp->negotiated_version = SMB_DIRECT_VERSION_LE;
1588 resp->reserved = 0;
1589 resp->credits_requested =
1590 cpu_to_le16(t->send_credit_target);
1591 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1592 resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size);
1593 resp->preferred_send_size = cpu_to_le32(t->max_send_size);
1594 resp->max_receive_size = cpu_to_le32(t->max_recv_size);
1595 resp->max_fragmented_size =
1596 cpu_to_le32(t->max_fragmented_recv_size);
1597 }
1598
1599 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1600 (void *)resp, sizeof(*resp),
1601 DMA_TO_DEVICE);
1602 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1603 if (ret) {
1604 smb_direct_free_sendmsg(t, sendmsg);
1605 return ret;
1606 }
1607
1608 sendmsg->num_sge = 1;
1609 sendmsg->sge[0].length = sizeof(*resp);
1610 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1611
1612 ret = post_sendmsg(t, NULL, sendmsg);
1613 if (ret) {
1614 smb_direct_free_sendmsg(t, sendmsg);
1615 return ret;
1616 }
1617
1618 wait_event(t->wait_send_pending,
1619 atomic_read(&t->send_pending) == 0);
1620 return 0;
1621 }
1622
smb_direct_accept_client(struct smb_direct_transport * t)1623 static int smb_direct_accept_client(struct smb_direct_transport *t)
1624 {
1625 struct rdma_conn_param conn_param;
1626 struct ib_port_immutable port_immutable;
1627 u32 ird_ord_hdr[2];
1628 int ret;
1629
1630 memset(&conn_param, 0, sizeof(conn_param));
1631 conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom,
1632 SMB_DIRECT_CM_INITIATOR_DEPTH);
1633 conn_param.responder_resources = 0;
1634
1635 t->cm_id->device->ops.get_port_immutable(t->cm_id->device,
1636 t->cm_id->port_num,
1637 &port_immutable);
1638 if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
1639 ird_ord_hdr[0] = conn_param.responder_resources;
1640 ird_ord_hdr[1] = 1;
1641 conn_param.private_data = ird_ord_hdr;
1642 conn_param.private_data_len = sizeof(ird_ord_hdr);
1643 } else {
1644 conn_param.private_data = NULL;
1645 conn_param.private_data_len = 0;
1646 }
1647 conn_param.retry_count = SMB_DIRECT_CM_RETRY;
1648 conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
1649 conn_param.flow_control = 0;
1650
1651 ret = rdma_accept(t->cm_id, &conn_param);
1652 if (ret) {
1653 pr_err("error at rdma_accept: %d\n", ret);
1654 return ret;
1655 }
1656 return 0;
1657 }
1658
smb_direct_prepare_negotiation(struct smb_direct_transport * t)1659 static int smb_direct_prepare_negotiation(struct smb_direct_transport *t)
1660 {
1661 int ret;
1662 struct smb_direct_recvmsg *recvmsg;
1663
1664 recvmsg = get_free_recvmsg(t);
1665 if (!recvmsg)
1666 return -ENOMEM;
1667 recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ;
1668
1669 ret = smb_direct_post_recv(t, recvmsg);
1670 if (ret) {
1671 pr_err("Can't post recv: %d\n", ret);
1672 goto out_err;
1673 }
1674
1675 t->negotiation_requested = false;
1676 ret = smb_direct_accept_client(t);
1677 if (ret) {
1678 pr_err("Can't accept client\n");
1679 goto out_err;
1680 }
1681
1682 smb_direct_post_recv_credits(&t->post_recv_credits_work);
1683 return 0;
1684 out_err:
1685 put_recvmsg(t, recvmsg);
1686 return ret;
1687 }
1688
smb_direct_get_max_fr_pages(struct smb_direct_transport * t)1689 static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t)
1690 {
1691 return min_t(unsigned int,
1692 t->cm_id->device->attrs.max_fast_reg_page_list_len,
1693 256);
1694 }
1695
smb_direct_init_params(struct smb_direct_transport * t,struct ib_qp_cap * cap)1696 static int smb_direct_init_params(struct smb_direct_transport *t,
1697 struct ib_qp_cap *cap)
1698 {
1699 struct ib_device *device = t->cm_id->device;
1700 int max_send_sges, max_rw_wrs, max_send_wrs;
1701 unsigned int max_sge_per_wr, wrs_per_credit;
1702
1703 /* need 3 more sge. because a SMB_DIRECT header, SMB2 header,
1704 * SMB2 response could be mapped.
1705 */
1706 t->max_send_size = smb_direct_max_send_size;
1707 max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 3;
1708 if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) {
1709 pr_err("max_send_size %d is too large\n", t->max_send_size);
1710 return -EINVAL;
1711 }
1712
1713 /* Calculate the number of work requests for RDMA R/W.
1714 * The maximum number of pages which can be registered
1715 * with one Memory region can be transferred with one
1716 * R/W credit. And at least 4 work requests for each credit
1717 * are needed for MR registration, RDMA R/W, local & remote
1718 * MR invalidation.
1719 */
1720 t->max_rdma_rw_size = smb_direct_max_read_write_size;
1721 t->pages_per_rw_credit = smb_direct_get_max_fr_pages(t);
1722 t->max_rw_credits = DIV_ROUND_UP(t->max_rdma_rw_size,
1723 (t->pages_per_rw_credit - 1) *
1724 PAGE_SIZE);
1725
1726 max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge,
1727 device->attrs.max_sge_rd);
1728 max_sge_per_wr = max_t(unsigned int, max_sge_per_wr,
1729 max_send_sges);
1730 wrs_per_credit = max_t(unsigned int, 4,
1731 DIV_ROUND_UP(t->pages_per_rw_credit,
1732 max_sge_per_wr) + 1);
1733 max_rw_wrs = t->max_rw_credits * wrs_per_credit;
1734
1735 max_send_wrs = smb_direct_send_credit_target + max_rw_wrs;
1736 if (max_send_wrs > device->attrs.max_cqe ||
1737 max_send_wrs > device->attrs.max_qp_wr) {
1738 pr_err("consider lowering send_credit_target = %d\n",
1739 smb_direct_send_credit_target);
1740 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
1741 device->attrs.max_cqe, device->attrs.max_qp_wr);
1742 return -EINVAL;
1743 }
1744
1745 if (smb_direct_receive_credit_max > device->attrs.max_cqe ||
1746 smb_direct_receive_credit_max > device->attrs.max_qp_wr) {
1747 pr_err("consider lowering receive_credit_max = %d\n",
1748 smb_direct_receive_credit_max);
1749 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
1750 device->attrs.max_cqe, device->attrs.max_qp_wr);
1751 return -EINVAL;
1752 }
1753
1754 if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
1755 pr_err("warning: device max_recv_sge = %d too small\n",
1756 device->attrs.max_recv_sge);
1757 return -EINVAL;
1758 }
1759
1760 t->recv_credits = 0;
1761 t->count_avail_recvmsg = 0;
1762
1763 t->recv_credit_max = smb_direct_receive_credit_max;
1764 t->recv_credit_target = 10;
1765 t->new_recv_credits = 0;
1766
1767 t->send_credit_target = smb_direct_send_credit_target;
1768 atomic_set(&t->send_credits, 0);
1769 atomic_set(&t->rw_credits, t->max_rw_credits);
1770
1771 t->max_send_size = smb_direct_max_send_size;
1772 t->max_recv_size = smb_direct_max_receive_size;
1773 t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size;
1774
1775 cap->max_send_wr = max_send_wrs;
1776 cap->max_recv_wr = t->recv_credit_max;
1777 cap->max_send_sge = max_sge_per_wr;
1778 cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
1779 cap->max_inline_data = 0;
1780 cap->max_rdma_ctxs = t->max_rw_credits;
1781 return 0;
1782 }
1783
smb_direct_destroy_pools(struct smb_direct_transport * t)1784 static void smb_direct_destroy_pools(struct smb_direct_transport *t)
1785 {
1786 struct smb_direct_recvmsg *recvmsg;
1787
1788 while ((recvmsg = get_free_recvmsg(t)))
1789 mempool_free(recvmsg, t->recvmsg_mempool);
1790
1791 mempool_destroy(t->recvmsg_mempool);
1792 t->recvmsg_mempool = NULL;
1793
1794 kmem_cache_destroy(t->recvmsg_cache);
1795 t->recvmsg_cache = NULL;
1796
1797 mempool_destroy(t->sendmsg_mempool);
1798 t->sendmsg_mempool = NULL;
1799
1800 kmem_cache_destroy(t->sendmsg_cache);
1801 t->sendmsg_cache = NULL;
1802 }
1803
smb_direct_create_pools(struct smb_direct_transport * t)1804 static int smb_direct_create_pools(struct smb_direct_transport *t)
1805 {
1806 char name[80];
1807 int i;
1808 struct smb_direct_recvmsg *recvmsg;
1809
1810 snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t);
1811 t->sendmsg_cache = kmem_cache_create(name,
1812 sizeof(struct smb_direct_sendmsg) +
1813 sizeof(struct smb_direct_negotiate_resp),
1814 0, SLAB_HWCACHE_ALIGN, NULL);
1815 if (!t->sendmsg_cache)
1816 return -ENOMEM;
1817
1818 t->sendmsg_mempool = mempool_create(t->send_credit_target,
1819 mempool_alloc_slab, mempool_free_slab,
1820 t->sendmsg_cache);
1821 if (!t->sendmsg_mempool)
1822 goto err;
1823
1824 snprintf(name, sizeof(name), "smb_direct_resp_%p", t);
1825 t->recvmsg_cache = kmem_cache_create(name,
1826 sizeof(struct smb_direct_recvmsg) +
1827 t->max_recv_size,
1828 0, SLAB_HWCACHE_ALIGN, NULL);
1829 if (!t->recvmsg_cache)
1830 goto err;
1831
1832 t->recvmsg_mempool =
1833 mempool_create(t->recv_credit_max, mempool_alloc_slab,
1834 mempool_free_slab, t->recvmsg_cache);
1835 if (!t->recvmsg_mempool)
1836 goto err;
1837
1838 INIT_LIST_HEAD(&t->recvmsg_queue);
1839
1840 for (i = 0; i < t->recv_credit_max; i++) {
1841 recvmsg = mempool_alloc(t->recvmsg_mempool, KSMBD_DEFAULT_GFP);
1842 if (!recvmsg)
1843 goto err;
1844 recvmsg->transport = t;
1845 recvmsg->sge.length = 0;
1846 list_add(&recvmsg->list, &t->recvmsg_queue);
1847 }
1848 t->count_avail_recvmsg = t->recv_credit_max;
1849
1850 return 0;
1851 err:
1852 smb_direct_destroy_pools(t);
1853 return -ENOMEM;
1854 }
1855
smb_direct_create_qpair(struct smb_direct_transport * t,struct ib_qp_cap * cap)1856 static int smb_direct_create_qpair(struct smb_direct_transport *t,
1857 struct ib_qp_cap *cap)
1858 {
1859 int ret;
1860 struct ib_qp_init_attr qp_attr;
1861 int pages_per_rw;
1862
1863 t->pd = ib_alloc_pd(t->cm_id->device, 0);
1864 if (IS_ERR(t->pd)) {
1865 pr_err("Can't create RDMA PD\n");
1866 ret = PTR_ERR(t->pd);
1867 t->pd = NULL;
1868 return ret;
1869 }
1870
1871 t->send_cq = ib_alloc_cq(t->cm_id->device, t,
1872 smb_direct_send_credit_target + cap->max_rdma_ctxs,
1873 0, IB_POLL_WORKQUEUE);
1874 if (IS_ERR(t->send_cq)) {
1875 pr_err("Can't create RDMA send CQ\n");
1876 ret = PTR_ERR(t->send_cq);
1877 t->send_cq = NULL;
1878 goto err;
1879 }
1880
1881 t->recv_cq = ib_alloc_cq(t->cm_id->device, t,
1882 t->recv_credit_max, 0, IB_POLL_WORKQUEUE);
1883 if (IS_ERR(t->recv_cq)) {
1884 pr_err("Can't create RDMA recv CQ\n");
1885 ret = PTR_ERR(t->recv_cq);
1886 t->recv_cq = NULL;
1887 goto err;
1888 }
1889
1890 memset(&qp_attr, 0, sizeof(qp_attr));
1891 qp_attr.event_handler = smb_direct_qpair_handler;
1892 qp_attr.qp_context = t;
1893 qp_attr.cap = *cap;
1894 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
1895 qp_attr.qp_type = IB_QPT_RC;
1896 qp_attr.send_cq = t->send_cq;
1897 qp_attr.recv_cq = t->recv_cq;
1898 qp_attr.port_num = ~0;
1899
1900 ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr);
1901 if (ret) {
1902 pr_err("Can't create RDMA QP: %d\n", ret);
1903 goto err;
1904 }
1905
1906 t->qp = t->cm_id->qp;
1907 t->cm_id->event_handler = smb_direct_cm_handler;
1908
1909 pages_per_rw = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1;
1910 if (pages_per_rw > t->cm_id->device->attrs.max_sgl_rd) {
1911 ret = ib_mr_pool_init(t->qp, &t->qp->rdma_mrs,
1912 t->max_rw_credits, IB_MR_TYPE_MEM_REG,
1913 t->pages_per_rw_credit, 0);
1914 if (ret) {
1915 pr_err("failed to init mr pool count %d pages %d\n",
1916 t->max_rw_credits, t->pages_per_rw_credit);
1917 goto err;
1918 }
1919 }
1920
1921 return 0;
1922 err:
1923 if (t->qp) {
1924 t->qp = NULL;
1925 rdma_destroy_qp(t->cm_id);
1926 }
1927 if (t->recv_cq) {
1928 ib_destroy_cq(t->recv_cq);
1929 t->recv_cq = NULL;
1930 }
1931 if (t->send_cq) {
1932 ib_destroy_cq(t->send_cq);
1933 t->send_cq = NULL;
1934 }
1935 if (t->pd) {
1936 ib_dealloc_pd(t->pd);
1937 t->pd = NULL;
1938 }
1939 return ret;
1940 }
1941
smb_direct_prepare(struct ksmbd_transport * t)1942 static int smb_direct_prepare(struct ksmbd_transport *t)
1943 {
1944 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1945 struct smb_direct_recvmsg *recvmsg;
1946 struct smb_direct_negotiate_req *req;
1947 int ret;
1948
1949 ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n");
1950 ret = wait_event_interruptible_timeout(st->wait_status,
1951 st->negotiation_requested ||
1952 st->status == SMB_DIRECT_CS_DISCONNECTED,
1953 SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ);
1954 if (ret <= 0 || st->status == SMB_DIRECT_CS_DISCONNECTED)
1955 return ret < 0 ? ret : -ETIMEDOUT;
1956
1957 recvmsg = get_first_reassembly(st);
1958 if (!recvmsg)
1959 return -ECONNABORTED;
1960
1961 ret = smb_direct_check_recvmsg(recvmsg);
1962 if (ret == -ECONNABORTED)
1963 goto out;
1964
1965 req = (struct smb_direct_negotiate_req *)recvmsg->packet;
1966 st->max_recv_size = min_t(int, st->max_recv_size,
1967 le32_to_cpu(req->preferred_send_size));
1968 st->max_send_size = min_t(int, st->max_send_size,
1969 le32_to_cpu(req->max_receive_size));
1970 st->max_fragmented_send_size =
1971 le32_to_cpu(req->max_fragmented_size);
1972 st->max_fragmented_recv_size =
1973 (st->recv_credit_max * st->max_recv_size) / 2;
1974
1975 ret = smb_direct_send_negotiate_response(st, ret);
1976 out:
1977 spin_lock_irq(&st->reassembly_queue_lock);
1978 st->reassembly_queue_length--;
1979 list_del(&recvmsg->list);
1980 spin_unlock_irq(&st->reassembly_queue_lock);
1981 put_recvmsg(st, recvmsg);
1982
1983 return ret;
1984 }
1985
smb_direct_connect(struct smb_direct_transport * st)1986 static int smb_direct_connect(struct smb_direct_transport *st)
1987 {
1988 int ret;
1989 struct ib_qp_cap qp_cap;
1990
1991 ret = smb_direct_init_params(st, &qp_cap);
1992 if (ret) {
1993 pr_err("Can't configure RDMA parameters\n");
1994 return ret;
1995 }
1996
1997 ret = smb_direct_create_pools(st);
1998 if (ret) {
1999 pr_err("Can't init RDMA pool: %d\n", ret);
2000 return ret;
2001 }
2002
2003 ret = smb_direct_create_qpair(st, &qp_cap);
2004 if (ret) {
2005 pr_err("Can't accept RDMA client: %d\n", ret);
2006 return ret;
2007 }
2008
2009 ret = smb_direct_prepare_negotiation(st);
2010 if (ret) {
2011 pr_err("Can't negotiate: %d\n", ret);
2012 return ret;
2013 }
2014 return 0;
2015 }
2016
rdma_frwr_is_supported(struct ib_device_attr * attrs)2017 static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)
2018 {
2019 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
2020 return false;
2021 if (attrs->max_fast_reg_page_list_len == 0)
2022 return false;
2023 return true;
2024 }
2025
smb_direct_handle_connect_request(struct rdma_cm_id * new_cm_id)2026 static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)
2027 {
2028 struct smb_direct_transport *t;
2029 struct task_struct *handler;
2030 int ret;
2031
2032 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) {
2033 ksmbd_debug(RDMA,
2034 "Fast Registration Work Requests is not supported. device capabilities=%llx\n",
2035 new_cm_id->device->attrs.device_cap_flags);
2036 return -EPROTONOSUPPORT;
2037 }
2038
2039 t = alloc_transport(new_cm_id);
2040 if (!t)
2041 return -ENOMEM;
2042
2043 ret = smb_direct_connect(t);
2044 if (ret)
2045 goto out_err;
2046
2047 handler = kthread_run(ksmbd_conn_handler_loop,
2048 KSMBD_TRANS(t)->conn, "ksmbd:r%u",
2049 smb_direct_port);
2050 if (IS_ERR(handler)) {
2051 ret = PTR_ERR(handler);
2052 pr_err("Can't start thread\n");
2053 goto out_err;
2054 }
2055
2056 return 0;
2057 out_err:
2058 free_transport(t);
2059 return ret;
2060 }
2061
smb_direct_listen_handler(struct rdma_cm_id * cm_id,struct rdma_cm_event * event)2062 static int smb_direct_listen_handler(struct rdma_cm_id *cm_id,
2063 struct rdma_cm_event *event)
2064 {
2065 switch (event->event) {
2066 case RDMA_CM_EVENT_CONNECT_REQUEST: {
2067 int ret = smb_direct_handle_connect_request(cm_id);
2068
2069 if (ret) {
2070 pr_err("Can't create transport: %d\n", ret);
2071 return ret;
2072 }
2073
2074 ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n",
2075 cm_id);
2076 break;
2077 }
2078 default:
2079 pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n",
2080 cm_id, rdma_event_msg(event->event), event->event);
2081 break;
2082 }
2083 return 0;
2084 }
2085
smb_direct_listen(int port)2086 static int smb_direct_listen(int port)
2087 {
2088 int ret;
2089 struct rdma_cm_id *cm_id;
2090 struct sockaddr_in sin = {
2091 .sin_family = AF_INET,
2092 .sin_addr.s_addr = htonl(INADDR_ANY),
2093 .sin_port = htons(port),
2094 };
2095
2096 cm_id = rdma_create_id(&init_net, smb_direct_listen_handler,
2097 &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC);
2098 if (IS_ERR(cm_id)) {
2099 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id));
2100 return PTR_ERR(cm_id);
2101 }
2102
2103 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
2104 if (ret) {
2105 pr_err("Can't bind: %d\n", ret);
2106 goto err;
2107 }
2108
2109 smb_direct_listener.cm_id = cm_id;
2110
2111 ret = rdma_listen(cm_id, 10);
2112 if (ret) {
2113 pr_err("Can't listen: %d\n", ret);
2114 goto err;
2115 }
2116 return 0;
2117 err:
2118 smb_direct_listener.cm_id = NULL;
2119 rdma_destroy_id(cm_id);
2120 return ret;
2121 }
2122
smb_direct_ib_client_add(struct ib_device * ib_dev)2123 static int smb_direct_ib_client_add(struct ib_device *ib_dev)
2124 {
2125 struct smb_direct_device *smb_dev;
2126
2127 /* Set 5445 port if device type is iWARP(No IB) */
2128 if (ib_dev->node_type != RDMA_NODE_IB_CA)
2129 smb_direct_port = SMB_DIRECT_PORT_IWARP;
2130
2131 if (!rdma_frwr_is_supported(&ib_dev->attrs))
2132 return 0;
2133
2134 smb_dev = kzalloc(sizeof(*smb_dev), KSMBD_DEFAULT_GFP);
2135 if (!smb_dev)
2136 return -ENOMEM;
2137 smb_dev->ib_dev = ib_dev;
2138
2139 write_lock(&smb_direct_device_lock);
2140 list_add(&smb_dev->list, &smb_direct_device_list);
2141 write_unlock(&smb_direct_device_lock);
2142
2143 ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name);
2144 return 0;
2145 }
2146
smb_direct_ib_client_remove(struct ib_device * ib_dev,void * client_data)2147 static void smb_direct_ib_client_remove(struct ib_device *ib_dev,
2148 void *client_data)
2149 {
2150 struct smb_direct_device *smb_dev, *tmp;
2151
2152 write_lock(&smb_direct_device_lock);
2153 list_for_each_entry_safe(smb_dev, tmp, &smb_direct_device_list, list) {
2154 if (smb_dev->ib_dev == ib_dev) {
2155 list_del(&smb_dev->list);
2156 kfree(smb_dev);
2157 break;
2158 }
2159 }
2160 write_unlock(&smb_direct_device_lock);
2161 }
2162
2163 static struct ib_client smb_direct_ib_client = {
2164 .name = "ksmbd_smb_direct_ib",
2165 .add = smb_direct_ib_client_add,
2166 .remove = smb_direct_ib_client_remove,
2167 };
2168
ksmbd_rdma_init(void)2169 int ksmbd_rdma_init(void)
2170 {
2171 int ret;
2172
2173 smb_direct_listener.cm_id = NULL;
2174
2175 ret = ib_register_client(&smb_direct_ib_client);
2176 if (ret) {
2177 pr_err("failed to ib_register_client\n");
2178 return ret;
2179 }
2180
2181 /* When a client is running out of send credits, the credits are
2182 * granted by the server's sending a packet using this queue.
2183 * This avoids the situation that a clients cannot send packets
2184 * for lack of credits
2185 */
2186 smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq",
2187 WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
2188 if (!smb_direct_wq)
2189 return -ENOMEM;
2190
2191 ret = smb_direct_listen(smb_direct_port);
2192 if (ret) {
2193 destroy_workqueue(smb_direct_wq);
2194 smb_direct_wq = NULL;
2195 pr_err("Can't listen: %d\n", ret);
2196 return ret;
2197 }
2198
2199 ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n",
2200 smb_direct_listener.cm_id);
2201 return 0;
2202 }
2203
ksmbd_rdma_stop_listening(void)2204 void ksmbd_rdma_stop_listening(void)
2205 {
2206 if (!smb_direct_listener.cm_id)
2207 return;
2208
2209 ib_unregister_client(&smb_direct_ib_client);
2210 rdma_destroy_id(smb_direct_listener.cm_id);
2211
2212 smb_direct_listener.cm_id = NULL;
2213 }
2214
ksmbd_rdma_destroy(void)2215 void ksmbd_rdma_destroy(void)
2216 {
2217 if (smb_direct_wq) {
2218 destroy_workqueue(smb_direct_wq);
2219 smb_direct_wq = NULL;
2220 }
2221 }
2222
ksmbd_rdma_capable_netdev(struct net_device * netdev)2223 bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
2224 {
2225 struct smb_direct_device *smb_dev;
2226 int i;
2227 bool rdma_capable = false;
2228
2229 read_lock(&smb_direct_device_lock);
2230 list_for_each_entry(smb_dev, &smb_direct_device_list, list) {
2231 for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) {
2232 struct net_device *ndev;
2233
2234 if (smb_dev->ib_dev->ops.get_netdev) {
2235 ndev = smb_dev->ib_dev->ops.get_netdev(
2236 smb_dev->ib_dev, i + 1);
2237 if (!ndev)
2238 continue;
2239
2240 if (ndev == netdev) {
2241 dev_put(ndev);
2242 rdma_capable = true;
2243 goto out;
2244 }
2245 dev_put(ndev);
2246 /* if ib_dev does not implement ops.get_netdev
2247 * check for matching infiniband GUID in hw_addr
2248 */
2249 } else if (netdev->type == ARPHRD_INFINIBAND) {
2250 struct netdev_hw_addr *ha;
2251 union ib_gid gid;
2252 u32 port_num;
2253 int ret;
2254
2255 netdev_hw_addr_list_for_each(
2256 ha, &netdev->dev_addrs) {
2257 memcpy(&gid, ha->addr + 4, sizeof(gid));
2258 ret = ib_find_gid(smb_dev->ib_dev, &gid,
2259 &port_num, NULL);
2260 if (!ret) {
2261 rdma_capable = true;
2262 goto out;
2263 }
2264 }
2265 }
2266 }
2267 }
2268 out:
2269 read_unlock(&smb_direct_device_lock);
2270
2271 if (rdma_capable == false) {
2272 struct ib_device *ibdev;
2273
2274 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
2275 if (ibdev) {
2276 if (rdma_frwr_is_supported(&ibdev->attrs))
2277 rdma_capable = true;
2278 ib_device_put(ibdev);
2279 }
2280 }
2281
2282 return rdma_capable;
2283 }
2284
2285 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
2286 .prepare = smb_direct_prepare,
2287 .disconnect = smb_direct_disconnect,
2288 .shutdown = smb_direct_shutdown,
2289 .writev = smb_direct_writev,
2290 .read = smb_direct_read,
2291 .rdma_read = smb_direct_rdma_read,
2292 .rdma_write = smb_direct_rdma_write,
2293 .free_transport = smb_direct_free_transport,
2294 };
2295