1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_VERBS_H
8 #define RXE_VERBS_H
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include "rxe_pool.h"
13 #include "rxe_task.h"
14 #include "rxe_hw_counters.h"
15
pkey_match(u16 key1,u16 key2)16 static inline int pkey_match(u16 key1, u16 key2)
17 {
18 return (((key1 & 0x7fff) != 0) &&
19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
21 }
22
23 /* Return >0 if psn_a > psn_b
24 * 0 if psn_a == psn_b
25 * <0 if psn_a < psn_b
26 */
psn_compare(u32 psn_a,u32 psn_b)27 static inline int psn_compare(u32 psn_a, u32 psn_b)
28 {
29 s32 diff;
30
31 diff = (psn_a - psn_b) << 8;
32 return diff;
33 }
34
35 struct rxe_ucontext {
36 struct ib_ucontext ibuc;
37 struct rxe_pool_elem elem;
38 };
39
40 struct rxe_pd {
41 struct ib_pd ibpd;
42 struct rxe_pool_elem elem;
43 };
44
45 struct rxe_ah {
46 struct ib_ah ibah;
47 struct rxe_pool_elem elem;
48 struct rxe_av av;
49 bool is_user;
50 int ah_num;
51 };
52
53 struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58 };
59
60 struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_elem elem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_dying;
67 bool is_user;
68 struct tasklet_struct comp_task;
69 atomic_t num_wq;
70 };
71
72 enum wqe_state {
73 wqe_state_posted,
74 wqe_state_processing,
75 wqe_state_pending,
76 wqe_state_done,
77 wqe_state_error,
78 };
79
80 struct rxe_sq {
81 int max_wr;
82 int max_sge;
83 int max_inline;
84 spinlock_t sq_lock; /* guard queue */
85 struct rxe_queue *queue;
86 };
87
88 struct rxe_rq {
89 int max_wr;
90 int max_sge;
91 spinlock_t producer_lock; /* guard queue producer */
92 spinlock_t consumer_lock; /* guard queue consumer */
93 struct rxe_queue *queue;
94 };
95
96 struct rxe_srq {
97 struct ib_srq ibsrq;
98 struct rxe_pool_elem elem;
99 struct rxe_pd *pd;
100 struct rxe_rq rq;
101 u32 srq_num;
102
103 int limit;
104 int error;
105 };
106
107 enum rxe_qp_state {
108 QP_STATE_RESET,
109 QP_STATE_INIT,
110 QP_STATE_READY,
111 QP_STATE_DRAIN, /* req only */
112 QP_STATE_DRAINED, /* req only */
113 QP_STATE_ERROR
114 };
115
116 struct rxe_req_info {
117 enum rxe_qp_state state;
118 int wqe_index;
119 u32 psn;
120 int opcode;
121 atomic_t rd_atomic;
122 int wait_fence;
123 int need_rd_atomic;
124 int wait_psn;
125 int need_retry;
126 int wait_for_rnr_timer;
127 int noack_pkts;
128 struct rxe_task task;
129 };
130
131 struct rxe_comp_info {
132 enum rxe_qp_state state;
133 u32 psn;
134 int opcode;
135 int timeout;
136 int timeout_retry;
137 int started_retry;
138 u32 retry_cnt;
139 u32 rnr_retry;
140 struct rxe_task task;
141 };
142
143 enum rdatm_res_state {
144 rdatm_res_state_next,
145 rdatm_res_state_new,
146 rdatm_res_state_replay,
147 };
148
149 struct resp_res {
150 int type;
151 int replay;
152 u32 first_psn;
153 u32 last_psn;
154 u32 cur_psn;
155 enum rdatm_res_state state;
156
157 union {
158 struct {
159 u64 orig_val;
160 } atomic;
161 struct {
162 u64 va_org;
163 u32 rkey;
164 u32 length;
165 u64 va;
166 u32 resid;
167 } read;
168 };
169 };
170
171 struct rxe_resp_info {
172 enum rxe_qp_state state;
173 u32 msn;
174 u32 psn;
175 u32 ack_psn;
176 int opcode;
177 int drop_msg;
178 int goto_error;
179 int sent_psn_nak;
180 enum ib_wc_status status;
181 u8 aeth_syndrome;
182
183 /* Receive only */
184 struct rxe_recv_wqe *wqe;
185
186 /* RDMA read / atomic only */
187 u64 va;
188 u64 offset;
189 struct rxe_mr *mr;
190 u32 resid;
191 u32 rkey;
192 u32 length;
193
194 /* SRQ only */
195 struct {
196 struct rxe_recv_wqe wqe;
197 struct ib_sge sge[RXE_MAX_SGE];
198 } srq_wqe;
199
200 /* Responder resources. It's a circular list where the oldest
201 * resource is dropped first.
202 */
203 struct resp_res *resources;
204 unsigned int res_head;
205 unsigned int res_tail;
206 struct resp_res *res;
207 struct rxe_task task;
208 };
209
210 struct rxe_qp {
211 struct ib_qp ibqp;
212 struct rxe_pool_elem elem;
213 struct ib_qp_attr attr;
214 unsigned int valid;
215 unsigned int mtu;
216 bool is_user;
217
218 struct rxe_pd *pd;
219 struct rxe_srq *srq;
220 struct rxe_cq *scq;
221 struct rxe_cq *rcq;
222
223 enum ib_sig_type sq_sig_type;
224
225 struct rxe_sq sq;
226 struct rxe_rq rq;
227
228 struct socket *sk;
229 u32 dst_cookie;
230 u16 src_port;
231
232 struct rxe_av pri_av;
233 struct rxe_av alt_av;
234
235 atomic_t mcg_num;
236
237 struct sk_buff_head req_pkts;
238 struct sk_buff_head resp_pkts;
239
240 struct rxe_req_info req;
241 struct rxe_comp_info comp;
242 struct rxe_resp_info resp;
243
244 atomic_t ssn;
245 atomic_t skb_out;
246 int need_req_skb;
247
248 /* Timer for retranmitting packet when ACKs have been lost. RC
249 * only. The requester sets it when it is not already
250 * started. The responder resets it whenever an ack is
251 * received.
252 */
253 struct timer_list retrans_timer;
254 u64 qp_timeout_jiffies;
255
256 /* Timer for handling RNR NAKS. */
257 struct timer_list rnr_nak_timer;
258
259 spinlock_t state_lock; /* guard requester and completer */
260
261 struct execute_work cleanup_work;
262 };
263
264 enum rxe_mr_state {
265 RXE_MR_STATE_INVALID,
266 RXE_MR_STATE_FREE,
267 RXE_MR_STATE_VALID,
268 };
269
270 enum rxe_mr_copy_dir {
271 RXE_TO_MR_OBJ,
272 RXE_FROM_MR_OBJ,
273 };
274
275 enum rxe_mr_lookup_type {
276 RXE_LOOKUP_LOCAL,
277 RXE_LOOKUP_REMOTE,
278 };
279
280 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
281
282 struct rxe_phys_buf {
283 u64 addr;
284 u64 size;
285 };
286
287 struct rxe_map {
288 struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
289 };
290
rkey_is_mw(u32 rkey)291 static inline int rkey_is_mw(u32 rkey)
292 {
293 u32 index = rkey >> 8;
294
295 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
296 }
297
298 struct rxe_mr {
299 struct rxe_pool_elem elem;
300 struct ib_mr ibmr;
301
302 struct ib_umem *umem;
303
304 u32 lkey;
305 u32 rkey;
306 enum rxe_mr_state state;
307 enum ib_mr_type type;
308 u32 offset;
309 int access;
310
311 int page_shift;
312 int page_mask;
313 int map_shift;
314 int map_mask;
315
316 u32 num_buf;
317 u32 nbuf;
318
319 u32 max_buf;
320 u32 num_map;
321
322 atomic_t num_mw;
323
324 struct rxe_map **map;
325 };
326
327 enum rxe_mw_state {
328 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
329 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
330 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
331 };
332
333 struct rxe_mw {
334 struct ib_mw ibmw;
335 struct rxe_pool_elem elem;
336 spinlock_t lock;
337 enum rxe_mw_state state;
338 struct rxe_qp *qp; /* Type 2 only */
339 struct rxe_mr *mr;
340 u32 rkey;
341 int access;
342 u64 addr;
343 u64 length;
344 };
345
346 struct rxe_mcg {
347 struct rb_node node;
348 struct kref ref_cnt;
349 struct rxe_dev *rxe;
350 struct list_head qp_list;
351 union ib_gid mgid;
352 atomic_t qp_num;
353 u32 qkey;
354 u16 pkey;
355 };
356
357 struct rxe_mca {
358 struct list_head qp_list;
359 struct rxe_qp *qp;
360 };
361
362 struct rxe_port {
363 struct ib_port_attr attr;
364 __be64 port_guid;
365 __be64 subnet_prefix;
366 spinlock_t port_lock; /* guard port */
367 unsigned int mtu_cap;
368 /* special QPs */
369 u32 qp_gsi_index;
370 };
371
372 struct rxe_dev {
373 struct ib_device ib_dev;
374 struct ib_device_attr attr;
375 int max_ucontext;
376 int max_inline_data;
377 struct mutex usdev_lock;
378
379 struct net_device *ndev;
380
381 struct rxe_pool uc_pool;
382 struct rxe_pool pd_pool;
383 struct rxe_pool ah_pool;
384 struct rxe_pool srq_pool;
385 struct rxe_pool qp_pool;
386 struct rxe_pool cq_pool;
387 struct rxe_pool mr_pool;
388 struct rxe_pool mw_pool;
389
390 /* multicast support */
391 spinlock_t mcg_lock;
392 struct rb_root mcg_tree;
393 atomic_t mcg_num;
394 atomic_t mcg_attach;
395
396 spinlock_t pending_lock; /* guard pending_mmaps */
397 struct list_head pending_mmaps;
398
399 spinlock_t mmap_offset_lock; /* guard mmap_offset */
400 u64 mmap_offset;
401
402 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
403
404 struct rxe_port port;
405 struct crypto_shash *tfm;
406 };
407
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters index)408 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
409 {
410 atomic64_inc(&rxe->stats_counters[index]);
411 }
412
to_rdev(struct ib_device * dev)413 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
414 {
415 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
416 }
417
to_ruc(struct ib_ucontext * uc)418 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
419 {
420 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
421 }
422
to_rpd(struct ib_pd * pd)423 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
424 {
425 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
426 }
427
to_rah(struct ib_ah * ah)428 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
429 {
430 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
431 }
432
to_rsrq(struct ib_srq * srq)433 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
434 {
435 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
436 }
437
to_rqp(struct ib_qp * qp)438 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
439 {
440 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
441 }
442
to_rcq(struct ib_cq * cq)443 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
444 {
445 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
446 }
447
to_rmr(struct ib_mr * mr)448 static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
449 {
450 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
451 }
452
to_rmw(struct ib_mw * mw)453 static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
454 {
455 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
456 }
457
rxe_ah_pd(struct rxe_ah * ah)458 static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
459 {
460 return to_rpd(ah->ibah.pd);
461 }
462
mr_pd(struct rxe_mr * mr)463 static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
464 {
465 return to_rpd(mr->ibmr.pd);
466 }
467
rxe_mw_pd(struct rxe_mw * mw)468 static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
469 {
470 return to_rpd(mw->ibmw.pd);
471 }
472
473 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
474
475 #endif /* RXE_VERBS_H */
476