1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_VERBS_H
8 #define RXE_VERBS_H
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include <rdma/rdma_user_rxe.h>
13 #include "rxe_pool.h"
14 #include "rxe_task.h"
15 #include "rxe_hw_counters.h"
16
pkey_match(u16 key1,u16 key2)17 static inline int pkey_match(u16 key1, u16 key2)
18 {
19 return (((key1 & 0x7fff) != 0) &&
20 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
21 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
22 }
23
24 /* Return >0 if psn_a > psn_b
25 * 0 if psn_a == psn_b
26 * <0 if psn_a < psn_b
27 */
psn_compare(u32 psn_a,u32 psn_b)28 static inline int psn_compare(u32 psn_a, u32 psn_b)
29 {
30 s32 diff;
31
32 diff = (psn_a - psn_b) << 8;
33 return diff;
34 }
35
36 struct rxe_ucontext {
37 struct ib_ucontext ibuc;
38 struct rxe_pool_entry pelem;
39 };
40
41 struct rxe_pd {
42 struct ib_pd ibpd;
43 struct rxe_pool_entry pelem;
44 };
45
46 struct rxe_ah {
47 struct ib_ah ibah;
48 struct rxe_pool_entry pelem;
49 struct rxe_pd *pd;
50 struct rxe_av av;
51 };
52
53 struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58 };
59
60 struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_entry pelem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_dying;
67 int is_user;
68 struct tasklet_struct comp_task;
69 };
70
71 enum wqe_state {
72 wqe_state_posted,
73 wqe_state_processing,
74 wqe_state_pending,
75 wqe_state_done,
76 wqe_state_error,
77 };
78
79 struct rxe_sq {
80 int max_wr;
81 int max_sge;
82 int max_inline;
83 spinlock_t sq_lock; /* guard queue */
84 struct rxe_queue *queue;
85 };
86
87 struct rxe_rq {
88 int max_wr;
89 int max_sge;
90 spinlock_t producer_lock; /* guard queue producer */
91 spinlock_t consumer_lock; /* guard queue consumer */
92 struct rxe_queue *queue;
93 };
94
95 struct rxe_srq {
96 struct ib_srq ibsrq;
97 struct rxe_pool_entry pelem;
98 struct rxe_pd *pd;
99 struct rxe_rq rq;
100 u32 srq_num;
101
102 int limit;
103 int error;
104 };
105
106 enum rxe_qp_state {
107 QP_STATE_RESET,
108 QP_STATE_INIT,
109 QP_STATE_READY,
110 QP_STATE_DRAIN, /* req only */
111 QP_STATE_DRAINED, /* req only */
112 QP_STATE_ERROR
113 };
114
115 struct rxe_req_info {
116 enum rxe_qp_state state;
117 int wqe_index;
118 u32 psn;
119 int opcode;
120 atomic_t rd_atomic;
121 int wait_fence;
122 int need_rd_atomic;
123 int wait_psn;
124 int need_retry;
125 int noack_pkts;
126 struct rxe_task task;
127 };
128
129 struct rxe_comp_info {
130 u32 psn;
131 int opcode;
132 int timeout;
133 int timeout_retry;
134 int started_retry;
135 u32 retry_cnt;
136 u32 rnr_retry;
137 struct rxe_task task;
138 };
139
140 enum rdatm_res_state {
141 rdatm_res_state_next,
142 rdatm_res_state_new,
143 rdatm_res_state_replay,
144 };
145
146 struct resp_res {
147 int type;
148 int replay;
149 u32 first_psn;
150 u32 last_psn;
151 u32 cur_psn;
152 enum rdatm_res_state state;
153
154 union {
155 struct {
156 struct sk_buff *skb;
157 } atomic;
158 struct {
159 struct rxe_mem *mr;
160 u64 va_org;
161 u32 rkey;
162 u32 length;
163 u64 va;
164 u32 resid;
165 } read;
166 };
167 };
168
169 struct rxe_resp_info {
170 enum rxe_qp_state state;
171 u32 msn;
172 u32 psn;
173 u32 ack_psn;
174 int opcode;
175 int drop_msg;
176 int goto_error;
177 int sent_psn_nak;
178 enum ib_wc_status status;
179 u8 aeth_syndrome;
180
181 /* Receive only */
182 struct rxe_recv_wqe *wqe;
183
184 /* RDMA read / atomic only */
185 u64 va;
186 struct rxe_mem *mr;
187 u32 resid;
188 u32 rkey;
189 u32 length;
190 u64 atomic_orig;
191
192 /* SRQ only */
193 struct {
194 struct rxe_recv_wqe wqe;
195 struct ib_sge sge[RXE_MAX_SGE];
196 } srq_wqe;
197
198 /* Responder resources. It's a circular list where the oldest
199 * resource is dropped first.
200 */
201 struct resp_res *resources;
202 unsigned int res_head;
203 unsigned int res_tail;
204 struct resp_res *res;
205 struct rxe_task task;
206 };
207
208 struct rxe_qp {
209 struct rxe_pool_entry pelem;
210 struct ib_qp ibqp;
211 struct ib_qp_attr attr;
212 unsigned int valid;
213 unsigned int mtu;
214 int is_user;
215
216 struct rxe_pd *pd;
217 struct rxe_srq *srq;
218 struct rxe_cq *scq;
219 struct rxe_cq *rcq;
220
221 enum ib_sig_type sq_sig_type;
222
223 struct rxe_sq sq;
224 struct rxe_rq rq;
225
226 struct socket *sk;
227 u32 dst_cookie;
228 u16 src_port;
229
230 struct rxe_av pri_av;
231 struct rxe_av alt_av;
232
233 /* list of mcast groups qp has joined (for cleanup) */
234 struct list_head grp_list;
235 spinlock_t grp_lock; /* guard grp_list */
236
237 struct sk_buff_head req_pkts;
238 struct sk_buff_head resp_pkts;
239 struct sk_buff_head send_pkts;
240
241 struct rxe_req_info req;
242 struct rxe_comp_info comp;
243 struct rxe_resp_info resp;
244
245 atomic_t ssn;
246 atomic_t skb_out;
247 int need_req_skb;
248
249 /* Timer for retranmitting packet when ACKs have been lost. RC
250 * only. The requester sets it when it is not already
251 * started. The responder resets it whenever an ack is
252 * received.
253 */
254 struct timer_list retrans_timer;
255 u64 qp_timeout_jiffies;
256
257 /* Timer for handling RNR NAKS. */
258 struct timer_list rnr_nak_timer;
259
260 spinlock_t state_lock; /* guard requester and completer */
261
262 struct execute_work cleanup_work;
263 };
264
265 enum rxe_mem_state {
266 RXE_MEM_STATE_ZOMBIE,
267 RXE_MEM_STATE_INVALID,
268 RXE_MEM_STATE_FREE,
269 RXE_MEM_STATE_VALID,
270 };
271
272 enum rxe_mem_type {
273 RXE_MEM_TYPE_NONE,
274 RXE_MEM_TYPE_DMA,
275 RXE_MEM_TYPE_MR,
276 RXE_MEM_TYPE_FMR,
277 RXE_MEM_TYPE_MW,
278 };
279
280 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
281
282 struct rxe_phys_buf {
283 u64 addr;
284 u64 size;
285 };
286
287 struct rxe_map {
288 struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
289 };
290
291 struct rxe_mem {
292 struct rxe_pool_entry pelem;
293 union {
294 struct ib_mr ibmr;
295 struct ib_mw ibmw;
296 };
297
298 struct ib_umem *umem;
299
300 enum rxe_mem_state state;
301 enum rxe_mem_type type;
302 u64 va;
303 u64 iova;
304 size_t length;
305 u32 offset;
306 int access;
307
308 int page_shift;
309 int page_mask;
310 int map_shift;
311 int map_mask;
312
313 u32 num_buf;
314 u32 nbuf;
315
316 u32 max_buf;
317 u32 num_map;
318
319 struct rxe_map **map;
320 };
321
322 struct rxe_mc_grp {
323 struct rxe_pool_entry pelem;
324 spinlock_t mcg_lock; /* guard group */
325 struct rxe_dev *rxe;
326 struct list_head qp_list;
327 union ib_gid mgid;
328 int num_qp;
329 u32 qkey;
330 u16 pkey;
331 };
332
333 struct rxe_mc_elem {
334 struct rxe_pool_entry pelem;
335 struct list_head qp_list;
336 struct list_head grp_list;
337 struct rxe_qp *qp;
338 struct rxe_mc_grp *grp;
339 };
340
341 struct rxe_port {
342 struct ib_port_attr attr;
343 __be64 port_guid;
344 __be64 subnet_prefix;
345 spinlock_t port_lock; /* guard port */
346 unsigned int mtu_cap;
347 /* special QPs */
348 u32 qp_smi_index;
349 u32 qp_gsi_index;
350 };
351
352 struct rxe_dev {
353 struct ib_device ib_dev;
354 struct ib_device_attr attr;
355 int max_ucontext;
356 int max_inline_data;
357 struct mutex usdev_lock;
358
359 struct net_device *ndev;
360
361 int xmit_errors;
362
363 struct rxe_pool uc_pool;
364 struct rxe_pool pd_pool;
365 struct rxe_pool ah_pool;
366 struct rxe_pool srq_pool;
367 struct rxe_pool qp_pool;
368 struct rxe_pool cq_pool;
369 struct rxe_pool mr_pool;
370 struct rxe_pool mw_pool;
371 struct rxe_pool mc_grp_pool;
372 struct rxe_pool mc_elem_pool;
373
374 spinlock_t pending_lock; /* guard pending_mmaps */
375 struct list_head pending_mmaps;
376
377 spinlock_t mmap_offset_lock; /* guard mmap_offset */
378 u64 mmap_offset;
379
380 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
381
382 struct rxe_port port;
383 struct crypto_shash *tfm;
384 };
385
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters index)386 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
387 {
388 atomic64_inc(&rxe->stats_counters[index]);
389 }
390
to_rdev(struct ib_device * dev)391 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
392 {
393 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
394 }
395
to_ruc(struct ib_ucontext * uc)396 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
397 {
398 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
399 }
400
to_rpd(struct ib_pd * pd)401 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
402 {
403 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
404 }
405
to_rah(struct ib_ah * ah)406 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
407 {
408 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
409 }
410
to_rsrq(struct ib_srq * srq)411 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
412 {
413 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
414 }
415
to_rqp(struct ib_qp * qp)416 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
417 {
418 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
419 }
420
to_rcq(struct ib_cq * cq)421 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
422 {
423 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
424 }
425
to_rmr(struct ib_mr * mr)426 static inline struct rxe_mem *to_rmr(struct ib_mr *mr)
427 {
428 return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL;
429 }
430
to_rmw(struct ib_mw * mw)431 static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
432 {
433 return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
434 }
435
mr_pd(struct rxe_mem * mr)436 static inline struct rxe_pd *mr_pd(struct rxe_mem *mr)
437 {
438 return to_rpd(mr->ibmr.pd);
439 }
440
mr_lkey(struct rxe_mem * mr)441 static inline u32 mr_lkey(struct rxe_mem *mr)
442 {
443 return mr->ibmr.lkey;
444 }
445
mr_rkey(struct rxe_mem * mr)446 static inline u32 mr_rkey(struct rxe_mem *mr)
447 {
448 return mr->ibmr.rkey;
449 }
450
451 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
452
453 void rxe_mc_cleanup(struct rxe_pool_entry *arg);
454
455 #endif /* RXE_VERBS_H */
456