1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_VERBS_H
8 #define RXE_VERBS_H
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include "rxe_pool.h"
13 #include "rxe_task.h"
14 #include "rxe_hw_counters.h"
15
pkey_match(u16 key1,u16 key2)16 static inline int pkey_match(u16 key1, u16 key2)
17 {
18 return (((key1 & 0x7fff) != 0) &&
19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
21 }
22
23 /* Return >0 if psn_a > psn_b
24 * 0 if psn_a == psn_b
25 * <0 if psn_a < psn_b
26 */
psn_compare(u32 psn_a,u32 psn_b)27 static inline int psn_compare(u32 psn_a, u32 psn_b)
28 {
29 s32 diff;
30
31 diff = (psn_a - psn_b) << 8;
32 return diff;
33 }
34
35 struct rxe_ucontext {
36 struct ib_ucontext ibuc;
37 struct rxe_pool_elem elem;
38 };
39
40 struct rxe_pd {
41 struct ib_pd ibpd;
42 struct rxe_pool_elem elem;
43 };
44
45 struct rxe_ah {
46 struct ib_ah ibah;
47 struct rxe_pool_elem elem;
48 struct rxe_av av;
49 bool is_user;
50 int ah_num;
51 };
52
53 struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58 };
59
60 struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_elem elem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_user;
67 atomic_t num_wq;
68 };
69
70 enum wqe_state {
71 wqe_state_posted,
72 wqe_state_processing,
73 wqe_state_pending,
74 wqe_state_done,
75 wqe_state_error,
76 };
77
78 struct rxe_sq {
79 int max_wr;
80 int max_sge;
81 int max_inline;
82 spinlock_t sq_lock; /* guard queue */
83 struct rxe_queue *queue;
84 };
85
86 struct rxe_rq {
87 int max_wr;
88 int max_sge;
89 spinlock_t producer_lock; /* guard queue producer */
90 spinlock_t consumer_lock; /* guard queue consumer */
91 struct rxe_queue *queue;
92 };
93
94 struct rxe_srq {
95 struct ib_srq ibsrq;
96 struct rxe_pool_elem elem;
97 struct rxe_pd *pd;
98 struct rxe_rq rq;
99 u32 srq_num;
100
101 int limit;
102 int error;
103 };
104
105 struct rxe_req_info {
106 int wqe_index;
107 u32 psn;
108 int opcode;
109 atomic_t rd_atomic;
110 int wait_fence;
111 int need_rd_atomic;
112 int wait_psn;
113 int need_retry;
114 int wait_for_rnr_timer;
115 int noack_pkts;
116 struct rxe_task task;
117 };
118
119 struct rxe_comp_info {
120 u32 psn;
121 int opcode;
122 int timeout;
123 int timeout_retry;
124 int started_retry;
125 u32 retry_cnt;
126 u32 rnr_retry;
127 struct rxe_task task;
128 };
129
130 enum rdatm_res_state {
131 rdatm_res_state_next,
132 rdatm_res_state_new,
133 rdatm_res_state_replay,
134 };
135
136 struct resp_res {
137 int type;
138 int replay;
139 u32 first_psn;
140 u32 last_psn;
141 u32 cur_psn;
142 enum rdatm_res_state state;
143
144 union {
145 struct {
146 u64 orig_val;
147 } atomic;
148 struct {
149 u64 va_org;
150 u32 rkey;
151 u32 length;
152 u64 va;
153 u32 resid;
154 } read;
155 struct {
156 u32 length;
157 u64 va;
158 u8 type;
159 u8 level;
160 } flush;
161 };
162 };
163
164 struct rxe_resp_info {
165 u32 msn;
166 u32 psn;
167 u32 ack_psn;
168 int opcode;
169 int drop_msg;
170 int goto_error;
171 int sent_psn_nak;
172 enum ib_wc_status status;
173 u8 aeth_syndrome;
174
175 /* Receive only */
176 struct rxe_recv_wqe *wqe;
177
178 /* RDMA read / atomic only */
179 u64 va;
180 u64 offset;
181 struct rxe_mr *mr;
182 u32 resid;
183 u32 rkey;
184 u32 length;
185
186 /* SRQ only */
187 struct {
188 struct rxe_recv_wqe wqe;
189 struct ib_sge sge[RXE_MAX_SGE];
190 } srq_wqe;
191
192 /* Responder resources. It's a circular list where the oldest
193 * resource is dropped first.
194 */
195 struct resp_res *resources;
196 unsigned int res_head;
197 unsigned int res_tail;
198 struct resp_res *res;
199 struct rxe_task task;
200 };
201
202 struct rxe_qp {
203 struct ib_qp ibqp;
204 struct rxe_pool_elem elem;
205 struct ib_qp_attr attr;
206 unsigned int valid;
207 unsigned int mtu;
208 bool is_user;
209
210 struct rxe_pd *pd;
211 struct rxe_srq *srq;
212 struct rxe_cq *scq;
213 struct rxe_cq *rcq;
214
215 enum ib_sig_type sq_sig_type;
216
217 struct rxe_sq sq;
218 struct rxe_rq rq;
219
220 struct socket *sk;
221 u32 dst_cookie;
222 u16 src_port;
223
224 struct rxe_av pri_av;
225 struct rxe_av alt_av;
226
227 atomic_t mcg_num;
228
229 struct sk_buff_head req_pkts;
230 struct sk_buff_head resp_pkts;
231
232 struct rxe_req_info req;
233 struct rxe_comp_info comp;
234 struct rxe_resp_info resp;
235
236 atomic_t ssn;
237 atomic_t skb_out;
238 int need_req_skb;
239
240 /* Timer for retranmitting packet when ACKs have been lost. RC
241 * only. The requester sets it when it is not already
242 * started. The responder resets it whenever an ack is
243 * received.
244 */
245 struct timer_list retrans_timer;
246 u64 qp_timeout_jiffies;
247
248 /* Timer for handling RNR NAKS. */
249 struct timer_list rnr_nak_timer;
250
251 spinlock_t state_lock; /* guard requester and completer */
252
253 struct execute_work cleanup_work;
254 };
255
256 enum {
257 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
258 | IB_ACCESS_REMOTE_WRITE
259 | IB_ACCESS_REMOTE_ATOMIC,
260 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
261 | IB_ACCESS_LOCAL_WRITE
262 | IB_ACCESS_MW_BIND
263 | IB_ACCESS_ON_DEMAND
264 | IB_ACCESS_FLUSH_GLOBAL
265 | IB_ACCESS_FLUSH_PERSISTENT
266 | IB_ACCESS_OPTIONAL,
267 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
268 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
269 | IB_ZERO_BASED,
270 };
271
272 enum rxe_mr_state {
273 RXE_MR_STATE_INVALID,
274 RXE_MR_STATE_FREE,
275 RXE_MR_STATE_VALID,
276 };
277
278 enum rxe_mr_copy_dir {
279 RXE_TO_MR_OBJ,
280 RXE_FROM_MR_OBJ,
281 };
282
283 enum rxe_mr_lookup_type {
284 RXE_LOOKUP_LOCAL,
285 RXE_LOOKUP_REMOTE,
286 };
287
288 enum rxe_rereg {
289 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
290 | IB_MR_REREG_ACCESS,
291 };
292
rkey_is_mw(u32 rkey)293 static inline int rkey_is_mw(u32 rkey)
294 {
295 u32 index = rkey >> 8;
296
297 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
298 }
299
300 struct rxe_mr {
301 struct rxe_pool_elem elem;
302 struct ib_mr ibmr;
303
304 struct ib_umem *umem;
305
306 u32 lkey;
307 u32 rkey;
308 enum rxe_mr_state state;
309 int access;
310 atomic_t num_mw;
311
312 unsigned int page_offset;
313 unsigned int page_shift;
314 u64 page_mask;
315
316 u32 num_buf;
317 u32 nbuf;
318
319 struct xarray page_list;
320 };
321
mr_page_size(struct rxe_mr * mr)322 static inline unsigned int mr_page_size(struct rxe_mr *mr)
323 {
324 return mr ? mr->ibmr.page_size : PAGE_SIZE;
325 }
326
327 enum rxe_mw_state {
328 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
329 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
330 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
331 };
332
333 struct rxe_mw {
334 struct ib_mw ibmw;
335 struct rxe_pool_elem elem;
336 spinlock_t lock;
337 enum rxe_mw_state state;
338 struct rxe_qp *qp; /* Type 2 only */
339 struct rxe_mr *mr;
340 u32 rkey;
341 int access;
342 u64 addr;
343 u64 length;
344 };
345
346 struct rxe_mcg {
347 struct rb_node node;
348 struct kref ref_cnt;
349 struct rxe_dev *rxe;
350 struct list_head qp_list;
351 union ib_gid mgid;
352 atomic_t qp_num;
353 u32 qkey;
354 u16 pkey;
355 };
356
357 struct rxe_mca {
358 struct list_head qp_list;
359 struct rxe_qp *qp;
360 };
361
362 struct rxe_port {
363 struct ib_port_attr attr;
364 __be64 port_guid;
365 __be64 subnet_prefix;
366 spinlock_t port_lock; /* guard port */
367 unsigned int mtu_cap;
368 /* special QPs */
369 u32 qp_gsi_index;
370 };
371
372 struct rxe_dev {
373 struct ib_device ib_dev;
374 struct ib_device_attr attr;
375 int max_ucontext;
376 int max_inline_data;
377 struct mutex usdev_lock;
378
379 struct net_device *ndev;
380
381 struct rxe_pool uc_pool;
382 struct rxe_pool pd_pool;
383 struct rxe_pool ah_pool;
384 struct rxe_pool srq_pool;
385 struct rxe_pool qp_pool;
386 struct rxe_pool cq_pool;
387 struct rxe_pool mr_pool;
388 struct rxe_pool mw_pool;
389
390 /* multicast support */
391 spinlock_t mcg_lock;
392 struct rb_root mcg_tree;
393 atomic_t mcg_num;
394 atomic_t mcg_attach;
395
396 spinlock_t pending_lock; /* guard pending_mmaps */
397 struct list_head pending_mmaps;
398
399 spinlock_t mmap_offset_lock; /* guard mmap_offset */
400 u64 mmap_offset;
401
402 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
403
404 struct rxe_port port;
405 struct crypto_shash *tfm;
406 };
407
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters index)408 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
409 {
410 atomic64_inc(&rxe->stats_counters[index]);
411 }
412
to_rdev(struct ib_device * dev)413 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
414 {
415 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
416 }
417
to_ruc(struct ib_ucontext * uc)418 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
419 {
420 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
421 }
422
to_rpd(struct ib_pd * pd)423 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
424 {
425 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
426 }
427
to_rah(struct ib_ah * ah)428 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
429 {
430 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
431 }
432
to_rsrq(struct ib_srq * srq)433 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
434 {
435 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
436 }
437
to_rqp(struct ib_qp * qp)438 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
439 {
440 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
441 }
442
to_rcq(struct ib_cq * cq)443 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
444 {
445 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
446 }
447
to_rmr(struct ib_mr * mr)448 static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
449 {
450 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
451 }
452
to_rmw(struct ib_mw * mw)453 static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
454 {
455 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
456 }
457
rxe_ah_pd(struct rxe_ah * ah)458 static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
459 {
460 return to_rpd(ah->ibah.pd);
461 }
462
mr_pd(struct rxe_mr * mr)463 static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
464 {
465 return to_rpd(mr->ibmr.pd);
466 }
467
rxe_mw_pd(struct rxe_mw * mw)468 static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
469 {
470 return to_rpd(mw->ibmw.pd);
471 }
472
473 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
474
475 #endif /* RXE_VERBS_H */
476