1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 #ifndef __ERDMA_VERBS_H__
8 #define __ERDMA_VERBS_H__
9
10 #include "erdma.h"
11
12 /* RDMA Capability. */
13 #define ERDMA_MAX_PD (128 * 1024)
14 #define ERDMA_MAX_SEND_WR 8192
15 #define ERDMA_MAX_ORD 128
16 #define ERDMA_MAX_IRD 128
17 #define ERDMA_MAX_SGE_RD 1
18 #define ERDMA_MAX_CONTEXT (128 * 1024)
19 #define ERDMA_MAX_SEND_SGE 6
20 #define ERDMA_MAX_RECV_SGE 1
21 #define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
22 #define ERDMA_MAX_FRMR_PA 512
23
24 enum {
25 ERDMA_MMAP_IO_NC = 0, /* no cache */
26 };
27
28 struct erdma_user_mmap_entry {
29 struct rdma_user_mmap_entry rdma_entry;
30 u64 address;
31 u8 mmap_flag;
32 };
33
34 struct erdma_ucontext {
35 struct ib_ucontext ibucontext;
36
37 u32 sdb_type;
38 u32 sdb_idx;
39 u32 sdb_page_idx;
40 u32 sdb_page_off;
41 u64 sdb;
42 u64 rdb;
43 u64 cdb;
44
45 struct rdma_user_mmap_entry *sq_db_mmap_entry;
46 struct rdma_user_mmap_entry *rq_db_mmap_entry;
47 struct rdma_user_mmap_entry *cq_db_mmap_entry;
48
49 /* doorbell records */
50 struct list_head dbrecords_page_list;
51 struct mutex dbrecords_page_mutex;
52 };
53
54 struct erdma_pd {
55 struct ib_pd ibpd;
56 u32 pdn;
57 };
58
59 /*
60 * MemoryRegion definition.
61 */
62 #define ERDMA_MAX_INLINE_MTT_ENTRIES 4
63 #define MTT_SIZE(mtt_cnt) (mtt_cnt << 3) /* per mtt takes 8 Bytes. */
64 #define ERDMA_MR_MAX_MTT_CNT 524288
65 #define ERDMA_MTT_ENTRY_SIZE 8
66
67 #define ERDMA_MR_TYPE_NORMAL 0
68 #define ERDMA_MR_TYPE_FRMR 1
69 #define ERDMA_MR_TYPE_DMA 2
70
71 #define ERDMA_MR_INLINE_MTT 0
72 #define ERDMA_MR_INDIRECT_MTT 1
73
74 #define ERDMA_MR_ACC_LR BIT(0)
75 #define ERDMA_MR_ACC_LW BIT(1)
76 #define ERDMA_MR_ACC_RR BIT(2)
77 #define ERDMA_MR_ACC_RW BIT(3)
78
to_erdma_access_flags(int access)79 static inline u8 to_erdma_access_flags(int access)
80 {
81 return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
82 (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
83 (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0);
84 }
85
86 struct erdma_mem {
87 struct ib_umem *umem;
88 void *mtt_buf;
89 u32 mtt_type;
90 u32 page_size;
91 u32 page_offset;
92 u32 page_cnt;
93 u32 mtt_nents;
94
95 u64 va;
96 u64 len;
97
98 u64 mtt_entry[ERDMA_MAX_INLINE_MTT_ENTRIES];
99 };
100
101 struct erdma_mr {
102 struct ib_mr ibmr;
103 struct erdma_mem mem;
104 u8 type;
105 u8 access;
106 u8 valid;
107 };
108
109 struct erdma_user_dbrecords_page {
110 struct list_head list;
111 struct ib_umem *umem;
112 u64 va;
113 int refcnt;
114 };
115
116 struct erdma_uqp {
117 struct erdma_mem sq_mtt;
118 struct erdma_mem rq_mtt;
119
120 dma_addr_t sq_db_info_dma_addr;
121 dma_addr_t rq_db_info_dma_addr;
122
123 struct erdma_user_dbrecords_page *user_dbr_page;
124
125 u32 rq_offset;
126 };
127
128 struct erdma_kqp {
129 u16 sq_pi;
130 u16 sq_ci;
131
132 u16 rq_pi;
133 u16 rq_ci;
134
135 u64 *swr_tbl;
136 u64 *rwr_tbl;
137
138 void __iomem *hw_sq_db;
139 void __iomem *hw_rq_db;
140
141 void *sq_buf;
142 dma_addr_t sq_buf_dma_addr;
143
144 void *rq_buf;
145 dma_addr_t rq_buf_dma_addr;
146
147 void *sq_db_info;
148 void *rq_db_info;
149
150 u8 sig_all;
151 };
152
153 enum erdma_qp_state {
154 ERDMA_QP_STATE_IDLE = 0,
155 ERDMA_QP_STATE_RTR = 1,
156 ERDMA_QP_STATE_RTS = 2,
157 ERDMA_QP_STATE_CLOSING = 3,
158 ERDMA_QP_STATE_TERMINATE = 4,
159 ERDMA_QP_STATE_ERROR = 5,
160 ERDMA_QP_STATE_UNDEF = 7,
161 ERDMA_QP_STATE_COUNT = 8
162 };
163
164 enum erdma_qp_attr_mask {
165 ERDMA_QP_ATTR_STATE = (1 << 0),
166 ERDMA_QP_ATTR_LLP_HANDLE = (1 << 2),
167 ERDMA_QP_ATTR_ORD = (1 << 3),
168 ERDMA_QP_ATTR_IRD = (1 << 4),
169 ERDMA_QP_ATTR_SQ_SIZE = (1 << 5),
170 ERDMA_QP_ATTR_RQ_SIZE = (1 << 6),
171 ERDMA_QP_ATTR_MPA = (1 << 7)
172 };
173
174 struct erdma_qp_attrs {
175 enum erdma_qp_state state;
176 enum erdma_cc_alg cc; /* Congestion control algorithm */
177 u32 sq_size;
178 u32 rq_size;
179 u32 orq_size;
180 u32 irq_size;
181 u32 max_send_sge;
182 u32 max_recv_sge;
183 u32 cookie;
184 #define ERDMA_QP_ACTIVE 0
185 #define ERDMA_QP_PASSIVE 1
186 u8 qp_type;
187 u8 pd_len;
188 };
189
190 struct erdma_qp {
191 struct ib_qp ibqp;
192 struct kref ref;
193 struct completion safe_free;
194 struct erdma_dev *dev;
195 struct erdma_cep *cep;
196 struct rw_semaphore state_lock;
197
198 union {
199 struct erdma_kqp kern_qp;
200 struct erdma_uqp user_qp;
201 };
202
203 struct erdma_cq *scq;
204 struct erdma_cq *rcq;
205
206 struct erdma_qp_attrs attrs;
207 spinlock_t lock;
208 };
209
210 struct erdma_kcq_info {
211 void *qbuf;
212 dma_addr_t qbuf_dma_addr;
213 u32 ci;
214 u32 cmdsn;
215 u32 notify_cnt;
216
217 spinlock_t lock;
218 u8 __iomem *db;
219 u64 *db_record;
220 };
221
222 struct erdma_ucq_info {
223 struct erdma_mem qbuf_mtt;
224 struct erdma_user_dbrecords_page *user_dbr_page;
225 dma_addr_t db_info_dma_addr;
226 };
227
228 struct erdma_cq {
229 struct ib_cq ibcq;
230 u32 cqn;
231
232 u32 depth;
233 u32 assoc_eqn;
234
235 union {
236 struct erdma_kcq_info kern_cq;
237 struct erdma_ucq_info user_cq;
238 };
239 };
240
241 #define QP_ID(qp) ((qp)->ibqp.qp_num)
242
find_qp_by_qpn(struct erdma_dev * dev,int id)243 static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
244 {
245 return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
246 }
247
find_cq_by_cqn(struct erdma_dev * dev,int id)248 static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
249 {
250 return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
251 }
252
253 void erdma_qp_get(struct erdma_qp *qp);
254 void erdma_qp_put(struct erdma_qp *qp);
255 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
256 enum erdma_qp_attr_mask mask);
257 void erdma_qp_llp_close(struct erdma_qp *qp);
258 void erdma_qp_cm_drop(struct erdma_qp *qp);
259
to_ectx(struct ib_ucontext * ibctx)260 static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
261 {
262 return container_of(ibctx, struct erdma_ucontext, ibucontext);
263 }
264
to_epd(struct ib_pd * pd)265 static inline struct erdma_pd *to_epd(struct ib_pd *pd)
266 {
267 return container_of(pd, struct erdma_pd, ibpd);
268 }
269
to_emr(struct ib_mr * ibmr)270 static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
271 {
272 return container_of(ibmr, struct erdma_mr, ibmr);
273 }
274
to_eqp(struct ib_qp * qp)275 static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
276 {
277 return container_of(qp, struct erdma_qp, ibqp);
278 }
279
to_ecq(struct ib_cq * ibcq)280 static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
281 {
282 return container_of(ibcq, struct erdma_cq, ibcq);
283 }
284
285 static inline struct erdma_user_mmap_entry *
to_emmap(struct rdma_user_mmap_entry * ibmmap)286 to_emmap(struct rdma_user_mmap_entry *ibmmap)
287 {
288 return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
289 }
290
291 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);
292 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);
293 int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
294 struct ib_udata *data);
295 int erdma_get_port_immutable(struct ib_device *dev, u32 port,
296 struct ib_port_immutable *ib_port_immutable);
297 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
298 struct ib_udata *data);
299 int erdma_query_port(struct ib_device *dev, u32 port,
300 struct ib_port_attr *attr);
301 int erdma_query_gid(struct ib_device *dev, u32 port, int idx,
302 union ib_gid *gid);
303 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);
304 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
305 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
306 struct ib_udata *data);
307 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
308 struct ib_qp_init_attr *init_attr);
309 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
310 struct ib_udata *data);
311 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
312 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
313 int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
314 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
315 u64 virt, int access, struct ib_udata *udata);
316 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
317 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
318 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
319 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
320 void erdma_qp_get_ref(struct ib_qp *ibqp);
321 void erdma_qp_put_ref(struct ib_qp *ibqp);
322 struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
323 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
324 const struct ib_send_wr **bad_send_wr);
325 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
326 const struct ib_recv_wr **bad_recv_wr);
327 int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
328 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
329 u32 max_num_sg);
330 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
331 unsigned int *sg_offset);
332 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
333 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu);
334
335 #endif
336