1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
4 */
5
6 /*
7 * The rdma_rxe driver supports type 1 or type 2B memory windows.
8 * Type 1 MWs are created by ibv_alloc_mw() verbs calls and bound by
9 * ibv_bind_mw() calls. Type 2 MWs are also created by ibv_alloc_mw()
10 * but bound by bind_mw work requests. The ibv_bind_mw() call is converted
11 * by libibverbs to a bind_mw work request.
12 */
13
14 #include "rxe.h"
15
rxe_alloc_mw(struct ib_mw * ibmw,struct ib_udata * udata)16 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
17 {
18 struct rxe_mw *mw = to_rmw(ibmw);
19 struct rxe_pd *pd = to_rpd(ibmw->pd);
20 struct rxe_dev *rxe = to_rdev(ibmw->device);
21 int ret;
22
23 rxe_get(pd);
24
25 ret = rxe_add_to_pool(&rxe->mw_pool, mw);
26 if (ret) {
27 rxe_put(pd);
28 return ret;
29 }
30
31 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1);
32 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
33 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
34 spin_lock_init(&mw->lock);
35
36 rxe_finalize(mw);
37
38 return 0;
39 }
40
rxe_dealloc_mw(struct ib_mw * ibmw)41 int rxe_dealloc_mw(struct ib_mw *ibmw)
42 {
43 struct rxe_mw *mw = to_rmw(ibmw);
44
45 rxe_cleanup(mw);
46
47 return 0;
48 }
49
rxe_check_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr,int access)50 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
51 struct rxe_mw *mw, struct rxe_mr *mr, int access)
52 {
53 if (mw->ibmw.type == IB_MW_TYPE_1) {
54 if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
55 rxe_dbg_mw(mw,
56 "attempt to bind a type 1 MW not in the valid state\n");
57 return -EINVAL;
58 }
59
60 /* o10-36.2.2 */
61 if (unlikely((access & IB_ZERO_BASED))) {
62 rxe_dbg_mw(mw, "attempt to bind a zero based type 1 MW\n");
63 return -EINVAL;
64 }
65 }
66
67 if (mw->ibmw.type == IB_MW_TYPE_2) {
68 /* o10-37.2.30 */
69 if (unlikely(mw->state != RXE_MW_STATE_FREE)) {
70 rxe_dbg_mw(mw,
71 "attempt to bind a type 2 MW not in the free state\n");
72 return -EINVAL;
73 }
74
75 /* C10-72 */
76 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
77 rxe_dbg_mw(mw,
78 "attempt to bind type 2 MW with qp with different PD\n");
79 return -EINVAL;
80 }
81
82 /* o10-37.2.40 */
83 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {
84 rxe_dbg_mw(mw,
85 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");
86 return -EINVAL;
87 }
88 }
89
90 /* remaining checks only apply to a nonzero MR */
91 if (!mr)
92 return 0;
93
94 if (unlikely(mr->access & IB_ZERO_BASED)) {
95 rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n");
96 return -EINVAL;
97 }
98
99 /* C10-73 */
100 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {
101 rxe_dbg_mw(mw,
102 "attempt to bind an MW to an MR without bind access\n");
103 return -EINVAL;
104 }
105
106 /* C10-74 */
107 if (unlikely((access &
108 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) &&
109 !(mr->access & IB_ACCESS_LOCAL_WRITE))) {
110 rxe_dbg_mw(mw,
111 "attempt to bind an Writable MW to an MR without local write access\n");
112 return -EINVAL;
113 }
114
115 /* C10-75 */
116 if (access & IB_ZERO_BASED) {
117 if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) {
118 rxe_dbg_mw(mw,
119 "attempt to bind a ZB MW outside of the MR\n");
120 return -EINVAL;
121 }
122 } else {
123 if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) ||
124 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >
125 (mr->ibmr.iova + mr->ibmr.length)))) {
126 rxe_dbg_mw(mw,
127 "attempt to bind a VA MW outside of the MR\n");
128 return -EINVAL;
129 }
130 }
131
132 return 0;
133 }
134
rxe_do_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr,int access)135 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
136 struct rxe_mw *mw, struct rxe_mr *mr, int access)
137 {
138 u32 key = wqe->wr.wr.mw.rkey & 0xff;
139
140 mw->rkey = (mw->rkey & ~0xff) | key;
141 mw->access = access;
142 mw->state = RXE_MW_STATE_VALID;
143 mw->addr = wqe->wr.wr.mw.addr;
144 mw->length = wqe->wr.wr.mw.length;
145
146 if (mw->mr) {
147 rxe_put(mw->mr);
148 atomic_dec(&mw->mr->num_mw);
149 mw->mr = NULL;
150 }
151
152 if (mw->length) {
153 mw->mr = mr;
154 atomic_inc(&mr->num_mw);
155 rxe_get(mr);
156 }
157
158 if (mw->ibmw.type == IB_MW_TYPE_2) {
159 rxe_get(qp);
160 mw->qp = qp;
161 }
162 }
163
rxe_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe)164 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
165 {
166 int ret;
167 struct rxe_mw *mw;
168 struct rxe_mr *mr;
169 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
170 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
171 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
172 int access = wqe->wr.wr.mw.access;
173
174 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
175 if (unlikely(!mw)) {
176 ret = -EINVAL;
177 goto err;
178 }
179
180 if (unlikely(mw->rkey != mw_rkey)) {
181 ret = -EINVAL;
182 goto err_drop_mw;
183 }
184
185 if (likely(wqe->wr.wr.mw.length)) {
186 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);
187 if (unlikely(!mr)) {
188 ret = -EINVAL;
189 goto err_drop_mw;
190 }
191
192 if (unlikely(mr->lkey != mr_lkey)) {
193 ret = -EINVAL;
194 goto err_drop_mr;
195 }
196 } else {
197 mr = NULL;
198 }
199
200 spin_lock_bh(&mw->lock);
201
202 ret = rxe_check_bind_mw(qp, wqe, mw, mr, access);
203 if (ret)
204 goto err_unlock;
205
206 rxe_do_bind_mw(qp, wqe, mw, mr, access);
207 err_unlock:
208 spin_unlock_bh(&mw->lock);
209 err_drop_mr:
210 if (mr)
211 rxe_put(mr);
212 err_drop_mw:
213 rxe_put(mw);
214 err:
215 return ret;
216 }
217
rxe_check_invalidate_mw(struct rxe_qp * qp,struct rxe_mw * mw)218 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw)
219 {
220 if (unlikely(mw->state == RXE_MW_STATE_INVALID))
221 return -EINVAL;
222
223 /* o10-37.2.26 */
224 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1))
225 return -EINVAL;
226
227 return 0;
228 }
229
rxe_do_invalidate_mw(struct rxe_mw * mw)230 static void rxe_do_invalidate_mw(struct rxe_mw *mw)
231 {
232 struct rxe_qp *qp;
233 struct rxe_mr *mr;
234
235 /* valid type 2 MW will always have a QP pointer */
236 qp = mw->qp;
237 mw->qp = NULL;
238 rxe_put(qp);
239
240 /* valid type 2 MW will always have an MR pointer */
241 mr = mw->mr;
242 mw->mr = NULL;
243 atomic_dec(&mr->num_mw);
244 rxe_put(mr);
245
246 mw->access = 0;
247 mw->addr = 0;
248 mw->length = 0;
249 mw->state = RXE_MW_STATE_FREE;
250 }
251
rxe_invalidate_mw(struct rxe_qp * qp,u32 rkey)252 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
253 {
254 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
255 struct rxe_mw *mw;
256 int ret;
257
258 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);
259 if (!mw) {
260 ret = -EINVAL;
261 goto err;
262 }
263
264 if (rkey != mw->rkey) {
265 ret = -EINVAL;
266 goto err_drop_ref;
267 }
268
269 spin_lock_bh(&mw->lock);
270
271 ret = rxe_check_invalidate_mw(qp, mw);
272 if (ret)
273 goto err_unlock;
274
275 rxe_do_invalidate_mw(mw);
276 err_unlock:
277 spin_unlock_bh(&mw->lock);
278 err_drop_ref:
279 rxe_put(mw);
280 err:
281 return ret;
282 }
283
rxe_lookup_mw(struct rxe_qp * qp,int access,u32 rkey)284 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
285 {
286 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
287 struct rxe_pd *pd = to_rpd(qp->ibqp.pd);
288 struct rxe_mw *mw;
289 int index = rkey >> 8;
290
291 mw = rxe_pool_get_index(&rxe->mw_pool, index);
292 if (!mw)
293 return NULL;
294
295 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
296 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
297 (mw->length == 0) ||
298 (access && !(access & mw->access)) ||
299 mw->state != RXE_MW_STATE_VALID)) {
300 rxe_put(mw);
301 return NULL;
302 }
303
304 return mw;
305 }
306
rxe_mw_cleanup(struct rxe_pool_elem * elem)307 void rxe_mw_cleanup(struct rxe_pool_elem *elem)
308 {
309 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem);
310 struct rxe_pd *pd = to_rpd(mw->ibmw.pd);
311
312 rxe_put(pd);
313
314 if (mw->mr) {
315 struct rxe_mr *mr = mw->mr;
316
317 mw->mr = NULL;
318 atomic_dec(&mr->num_mw);
319 rxe_put(mr);
320 }
321
322 if (mw->qp) {
323 struct rxe_qp *qp = mw->qp;
324
325 mw->qp = NULL;
326 rxe_put(qp);
327 }
328
329 mw->access = 0;
330 mw->addr = 0;
331 mw->length = 0;
332 mw->state = RXE_MW_STATE_INVALID;
333 }
334