1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "osdep.h"
4 #include "status.h"
5 #include "hmc.h"
6 #include "defs.h"
7 #include "type.h"
8 #include "protos.h"
9 #include "puda.h"
10 #include "ws.h"
11
12 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
13 struct irdma_puda_buf *buf);
14 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
15 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
16 struct irdma_puda_buf *buf, u32 wqe_idx);
17 /**
18 * irdma_puda_get_listbuf - get buffer from puda list
19 * @list: list to use for buffers (ILQ or IEQ)
20 */
irdma_puda_get_listbuf(struct list_head * list)21 static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)
22 {
23 struct irdma_puda_buf *buf = NULL;
24
25 if (!list_empty(list)) {
26 buf = (struct irdma_puda_buf *)list->next;
27 list_del((struct list_head *)&buf->list);
28 }
29
30 return buf;
31 }
32
33 /**
34 * irdma_puda_get_bufpool - return buffer from resource
35 * @rsrc: resource to use for buffer
36 */
irdma_puda_get_bufpool(struct irdma_puda_rsrc * rsrc)37 struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)
38 {
39 struct irdma_puda_buf *buf = NULL;
40 struct list_head *list = &rsrc->bufpool;
41 unsigned long flags;
42
43 spin_lock_irqsave(&rsrc->bufpool_lock, flags);
44 buf = irdma_puda_get_listbuf(list);
45 if (buf) {
46 rsrc->avail_buf_count--;
47 buf->vsi = rsrc->vsi;
48 } else {
49 rsrc->stats_buf_alloc_fail++;
50 }
51 spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
52
53 return buf;
54 }
55
56 /**
57 * irdma_puda_ret_bufpool - return buffer to rsrc list
58 * @rsrc: resource to use for buffer
59 * @buf: buffer to return to resource
60 */
irdma_puda_ret_bufpool(struct irdma_puda_rsrc * rsrc,struct irdma_puda_buf * buf)61 void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
62 struct irdma_puda_buf *buf)
63 {
64 unsigned long flags;
65
66 buf->do_lpb = false;
67 spin_lock_irqsave(&rsrc->bufpool_lock, flags);
68 list_add(&buf->list, &rsrc->bufpool);
69 spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
70 rsrc->avail_buf_count++;
71 }
72
73 /**
74 * irdma_puda_post_recvbuf - set wqe for rcv buffer
75 * @rsrc: resource ptr
76 * @wqe_idx: wqe index to use
77 * @buf: puda buffer for rcv q
78 * @initial: flag if during init time
79 */
irdma_puda_post_recvbuf(struct irdma_puda_rsrc * rsrc,u32 wqe_idx,struct irdma_puda_buf * buf,bool initial)80 static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
81 struct irdma_puda_buf *buf, bool initial)
82 {
83 __le64 *wqe;
84 struct irdma_sc_qp *qp = &rsrc->qp;
85 u64 offset24 = 0;
86
87 /* Synch buffer for use by device */
88 dma_sync_single_for_device(rsrc->dev->hw->device, buf->mem.pa,
89 buf->mem.size, DMA_BIDIRECTIONAL);
90 qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
91 wqe = qp->qp_uk.rq_base[wqe_idx].elem;
92 if (!initial)
93 get_64bit_val(wqe, 24, &offset24);
94
95 offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);
96
97 set_64bit_val(wqe, 16, 0);
98 set_64bit_val(wqe, 0, buf->mem.pa);
99 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
100 set_64bit_val(wqe, 8,
101 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf->mem.size));
102 } else {
103 set_64bit_val(wqe, 8,
104 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf->mem.size) |
105 offset24);
106 }
107 dma_wmb(); /* make sure WQE is written before valid bit is set */
108
109 set_64bit_val(wqe, 24, offset24);
110 }
111
112 /**
113 * irdma_puda_replenish_rq - post rcv buffers
114 * @rsrc: resource to use for buffer
115 * @initial: flag if during init time
116 */
117 static enum irdma_status_code
irdma_puda_replenish_rq(struct irdma_puda_rsrc * rsrc,bool initial)118 irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
119 {
120 u32 i;
121 u32 invalid_cnt = rsrc->rxq_invalid_cnt;
122 struct irdma_puda_buf *buf = NULL;
123
124 for (i = 0; i < invalid_cnt; i++) {
125 buf = irdma_puda_get_bufpool(rsrc);
126 if (!buf)
127 return IRDMA_ERR_list_empty;
128 irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
129 rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
130 rsrc->rxq_invalid_cnt--;
131 }
132
133 return 0;
134 }
135
136 /**
137 * irdma_puda_alloc_buf - allocate mem for buffer
138 * @dev: iwarp device
139 * @len: length of buffer
140 */
irdma_puda_alloc_buf(struct irdma_sc_dev * dev,u32 len)141 static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
142 u32 len)
143 {
144 struct irdma_puda_buf *buf;
145 struct irdma_virt_mem buf_mem;
146
147 buf_mem.size = sizeof(struct irdma_puda_buf);
148 buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
149 if (!buf_mem.va)
150 return NULL;
151
152 buf = buf_mem.va;
153 buf->mem.size = len;
154 buf->mem.va = kzalloc(buf->mem.size, GFP_KERNEL);
155 if (!buf->mem.va)
156 goto free_virt;
157 buf->mem.pa = dma_map_single(dev->hw->device, buf->mem.va,
158 buf->mem.size, DMA_BIDIRECTIONAL);
159 if (dma_mapping_error(dev->hw->device, buf->mem.pa)) {
160 kfree(buf->mem.va);
161 goto free_virt;
162 }
163
164 buf->buf_mem.va = buf_mem.va;
165 buf->buf_mem.size = buf_mem.size;
166
167 return buf;
168
169 free_virt:
170 kfree(buf_mem.va);
171 return NULL;
172 }
173
174 /**
175 * irdma_puda_dele_buf - delete buffer back to system
176 * @dev: iwarp device
177 * @buf: buffer to free
178 */
irdma_puda_dele_buf(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)179 static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,
180 struct irdma_puda_buf *buf)
181 {
182 dma_unmap_single(dev->hw->device, buf->mem.pa, buf->mem.size,
183 DMA_BIDIRECTIONAL);
184 kfree(buf->mem.va);
185 kfree(buf->buf_mem.va);
186 }
187
188 /**
189 * irdma_puda_get_next_send_wqe - return next wqe for processing
190 * @qp: puda qp for wqe
191 * @wqe_idx: wqe index for caller
192 */
irdma_puda_get_next_send_wqe(struct irdma_qp_uk * qp,u32 * wqe_idx)193 static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
194 u32 *wqe_idx)
195 {
196 __le64 *wqe = NULL;
197 enum irdma_status_code ret_code = 0;
198
199 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
200 if (!*wqe_idx)
201 qp->swqe_polarity = !qp->swqe_polarity;
202 IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code);
203 if (ret_code)
204 return wqe;
205
206 wqe = qp->sq_base[*wqe_idx].elem;
207
208 return wqe;
209 }
210
211 /**
212 * irdma_puda_poll_info - poll cq for completion
213 * @cq: cq for poll
214 * @info: info return for successful completion
215 */
216 static enum irdma_status_code
irdma_puda_poll_info(struct irdma_sc_cq * cq,struct irdma_puda_cmpl_info * info)217 irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
218 {
219 struct irdma_cq_uk *cq_uk = &cq->cq_uk;
220 u64 qword0, qword2, qword3, qword6;
221 __le64 *cqe;
222 __le64 *ext_cqe = NULL;
223 u64 qword7 = 0;
224 u64 comp_ctx;
225 bool valid_bit;
226 bool ext_valid = 0;
227 u32 major_err, minor_err;
228 u32 peek_head;
229 bool error;
230 u8 polarity;
231
232 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
233 get_64bit_val(cqe, 24, &qword3);
234 valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
235 if (valid_bit != cq_uk->polarity)
236 return IRDMA_ERR_Q_EMPTY;
237
238 /* Ensure CQE contents are read after valid bit is checked */
239 dma_rmb();
240
241 if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
242 ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
243
244 if (ext_valid) {
245 peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
246 ext_cqe = cq_uk->cq_base[peek_head].buf;
247 get_64bit_val(ext_cqe, 24, &qword7);
248 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
249 if (!peek_head)
250 polarity ^= 1;
251 if (polarity != cq_uk->polarity)
252 return IRDMA_ERR_Q_EMPTY;
253
254 /* Ensure ext CQE contents are read after ext valid bit is checked */
255 dma_rmb();
256
257 IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
258 if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
259 cq_uk->polarity = !cq_uk->polarity;
260 /* update cq tail in cq shadow memory also */
261 IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
262 }
263
264 print_hex_dump_debug("PUDA: PUDA CQE", DUMP_PREFIX_OFFSET, 16, 8, cqe,
265 32, false);
266 if (ext_valid)
267 print_hex_dump_debug("PUDA: PUDA EXT-CQE", DUMP_PREFIX_OFFSET,
268 16, 8, ext_cqe, 32, false);
269
270 error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
271 if (error) {
272 ibdev_dbg(to_ibdev(cq->dev), "PUDA: receive error\n");
273 major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
274 minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
275 info->compl_error = major_err << 16 | minor_err;
276 return IRDMA_ERR_CQ_COMPL_ERROR;
277 }
278
279 get_64bit_val(cqe, 0, &qword0);
280 get_64bit_val(cqe, 16, &qword2);
281
282 info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
283 info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
284 if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
285 info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
286
287 get_64bit_val(cqe, 8, &comp_ctx);
288 info->qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
289 info->wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
290
291 if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
292 if (ext_valid) {
293 info->vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
294 if (info->vlan_valid) {
295 get_64bit_val(ext_cqe, 16, &qword6);
296 info->vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
297 }
298 info->smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
299 if (info->smac_valid) {
300 get_64bit_val(ext_cqe, 16, &qword6);
301 info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
302 info->smac[1] = (u8)((qword6 >> 32) & 0xFF);
303 info->smac[2] = (u8)((qword6 >> 24) & 0xFF);
304 info->smac[3] = (u8)((qword6 >> 16) & 0xFF);
305 info->smac[4] = (u8)((qword6 >> 8) & 0xFF);
306 info->smac[5] = (u8)(qword6 & 0xFF);
307 }
308 }
309
310 if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
311 info->vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);
312 info->l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);
313 info->l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);
314 }
315
316 info->payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
317 }
318
319 return 0;
320 }
321
322 /**
323 * irdma_puda_poll_cmpl - processes completion for cq
324 * @dev: iwarp device
325 * @cq: cq getting interrupt
326 * @compl_err: return any completion err
327 */
irdma_puda_poll_cmpl(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq,u32 * compl_err)328 enum irdma_status_code irdma_puda_poll_cmpl(struct irdma_sc_dev *dev,
329 struct irdma_sc_cq *cq,
330 u32 *compl_err)
331 {
332 struct irdma_qp_uk *qp;
333 struct irdma_cq_uk *cq_uk = &cq->cq_uk;
334 struct irdma_puda_cmpl_info info = {};
335 enum irdma_status_code ret = 0;
336 struct irdma_puda_buf *buf;
337 struct irdma_puda_rsrc *rsrc;
338 u8 cq_type = cq->cq_type;
339 unsigned long flags;
340
341 if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {
342 rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :
343 cq->vsi->ieq;
344 } else {
345 ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");
346 return IRDMA_ERR_BAD_PTR;
347 }
348
349 ret = irdma_puda_poll_info(cq, &info);
350 *compl_err = info.compl_error;
351 if (ret == IRDMA_ERR_Q_EMPTY)
352 return ret;
353 if (ret)
354 goto done;
355
356 qp = info.qp;
357 if (!qp || !rsrc) {
358 ret = IRDMA_ERR_BAD_PTR;
359 goto done;
360 }
361
362 if (qp->qp_id != rsrc->qp_id) {
363 ret = IRDMA_ERR_BAD_PTR;
364 goto done;
365 }
366
367 if (info.q_type == IRDMA_CQE_QTYPE_RQ) {
368 buf = (struct irdma_puda_buf *)(uintptr_t)
369 qp->rq_wrid_array[info.wqe_idx];
370
371 /* reusing so synch the buffer for CPU use */
372 dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
373 buf->mem.size, DMA_BIDIRECTIONAL);
374 /* Get all the tcpip information in the buf header */
375 ret = irdma_puda_get_tcpip_info(&info, buf);
376 if (ret) {
377 rsrc->stats_rcvd_pkt_err++;
378 if (cq_type == IRDMA_CQ_TYPE_ILQ) {
379 irdma_ilq_putback_rcvbuf(&rsrc->qp, buf,
380 info.wqe_idx);
381 } else {
382 irdma_puda_ret_bufpool(rsrc, buf);
383 irdma_puda_replenish_rq(rsrc, false);
384 }
385 goto done;
386 }
387
388 rsrc->stats_pkt_rcvd++;
389 rsrc->compl_rxwqe_idx = info.wqe_idx;
390 ibdev_dbg(to_ibdev(dev), "PUDA: RQ completion\n");
391 rsrc->receive(rsrc->vsi, buf);
392 if (cq_type == IRDMA_CQ_TYPE_ILQ)
393 irdma_ilq_putback_rcvbuf(&rsrc->qp, buf, info.wqe_idx);
394 else
395 irdma_puda_replenish_rq(rsrc, false);
396
397 } else {
398 ibdev_dbg(to_ibdev(dev), "PUDA: SQ completion\n");
399 buf = (struct irdma_puda_buf *)(uintptr_t)
400 qp->sq_wrtrk_array[info.wqe_idx].wrid;
401
402 /* reusing so synch the buffer for CPU use */
403 dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
404 buf->mem.size, DMA_BIDIRECTIONAL);
405 IRDMA_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
406 rsrc->xmit_complete(rsrc->vsi, buf);
407 spin_lock_irqsave(&rsrc->bufpool_lock, flags);
408 rsrc->tx_wqe_avail_cnt++;
409 spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
410 if (!list_empty(&rsrc->txpend))
411 irdma_puda_send_buf(rsrc, NULL);
412 }
413
414 done:
415 IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
416 if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
417 cq_uk->polarity = !cq_uk->polarity;
418 /* update cq tail in cq shadow memory also */
419 IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
420 set_64bit_val(cq_uk->shadow_area, 0,
421 IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring));
422
423 return ret;
424 }
425
426 /**
427 * irdma_puda_send - complete send wqe for transmit
428 * @qp: puda qp for send
429 * @info: buffer information for transmit
430 */
irdma_puda_send(struct irdma_sc_qp * qp,struct irdma_puda_send_info * info)431 enum irdma_status_code irdma_puda_send(struct irdma_sc_qp *qp,
432 struct irdma_puda_send_info *info)
433 {
434 __le64 *wqe;
435 u32 iplen, l4len;
436 u64 hdr[2];
437 u32 wqe_idx;
438 u8 iipt;
439
440 /* number of 32 bits DWORDS in header */
441 l4len = info->tcplen >> 2;
442 if (info->ipv4) {
443 iipt = 3;
444 iplen = 5;
445 } else {
446 iipt = 1;
447 iplen = 10;
448 }
449
450 wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
451 if (!wqe)
452 return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
453
454 qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
455 /* Third line of WQE descriptor */
456 /* maclen is in words */
457
458 if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
459 hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
460 hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
461 FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |
462 FIELD_PREP(IRDMAQPSQ_AHID, info->ah_id) |
463 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
464 FIELD_PREP(IRDMA_UDA_QPSQ_VALID,
465 qp->qp_uk.swqe_polarity);
466
467 /* Forth line of WQE descriptor */
468
469 set_64bit_val(wqe, 0, info->paddr);
470 set_64bit_val(wqe, 8,
471 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info->len) |
472 FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity));
473 } else {
474 hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info->maclen >> 1) |
475 FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |
476 FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |
477 FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |
478 FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);
479
480 hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
481 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
482 FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info->do_lpb) |
483 FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity);
484
485 /* Forth line of WQE descriptor */
486
487 set_64bit_val(wqe, 0, info->paddr);
488 set_64bit_val(wqe, 8,
489 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info->len));
490 }
491
492 set_64bit_val(wqe, 16, hdr[0]);
493 dma_wmb(); /* make sure WQE is written before valid bit is set */
494
495 set_64bit_val(wqe, 24, hdr[1]);
496
497 print_hex_dump_debug("PUDA: PUDA SEND WQE", DUMP_PREFIX_OFFSET, 16, 8,
498 wqe, 32, false);
499 irdma_uk_qp_post_wr(&qp->qp_uk);
500 return 0;
501 }
502
503 /**
504 * irdma_puda_send_buf - transmit puda buffer
505 * @rsrc: resource to use for buffer
506 * @buf: puda buffer to transmit
507 */
irdma_puda_send_buf(struct irdma_puda_rsrc * rsrc,struct irdma_puda_buf * buf)508 void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
509 struct irdma_puda_buf *buf)
510 {
511 struct irdma_puda_send_info info;
512 enum irdma_status_code ret = 0;
513 unsigned long flags;
514
515 spin_lock_irqsave(&rsrc->bufpool_lock, flags);
516 /* if no wqe available or not from a completion and we have
517 * pending buffers, we must queue new buffer
518 */
519 if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
520 list_add_tail(&buf->list, &rsrc->txpend);
521 spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
522 rsrc->stats_sent_pkt_q++;
523 if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
524 ibdev_dbg(to_ibdev(rsrc->dev),
525 "PUDA: adding to txpend\n");
526 return;
527 }
528 rsrc->tx_wqe_avail_cnt--;
529 /* if we are coming from a completion and have pending buffers
530 * then Get one from pending list
531 */
532 if (!buf) {
533 buf = irdma_puda_get_listbuf(&rsrc->txpend);
534 if (!buf)
535 goto done;
536 }
537
538 info.scratch = buf;
539 info.paddr = buf->mem.pa;
540 info.len = buf->totallen;
541 info.tcplen = buf->tcphlen;
542 info.ipv4 = buf->ipv4;
543
544 if (rsrc->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
545 info.ah_id = buf->ah_id;
546 } else {
547 info.maclen = buf->maclen;
548 info.do_lpb = buf->do_lpb;
549 }
550
551 /* Synch buffer for use by device */
552 dma_sync_single_for_cpu(rsrc->dev->hw->device, buf->mem.pa,
553 buf->mem.size, DMA_BIDIRECTIONAL);
554 ret = irdma_puda_send(&rsrc->qp, &info);
555 if (ret) {
556 rsrc->tx_wqe_avail_cnt++;
557 rsrc->stats_sent_pkt_q++;
558 list_add(&buf->list, &rsrc->txpend);
559 if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
560 ibdev_dbg(to_ibdev(rsrc->dev),
561 "PUDA: adding to puda_send\n");
562 } else {
563 rsrc->stats_pkt_sent++;
564 }
565 done:
566 spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
567 }
568
569 /**
570 * irdma_puda_qp_setctx - during init, set qp's context
571 * @rsrc: qp's resource
572 */
irdma_puda_qp_setctx(struct irdma_puda_rsrc * rsrc)573 static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
574 {
575 struct irdma_sc_qp *qp = &rsrc->qp;
576 __le64 *qp_ctx = qp->hw_host_ctx;
577
578 set_64bit_val(qp_ctx, 8, qp->sq_pa);
579 set_64bit_val(qp_ctx, 16, qp->rq_pa);
580 set_64bit_val(qp_ctx, 24,
581 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
582 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size));
583 set_64bit_val(qp_ctx, 48,
584 FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));
585 set_64bit_val(qp_ctx, 56, 0);
586 if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
587 set_64bit_val(qp_ctx, 64, 1);
588 set_64bit_val(qp_ctx, 136,
589 FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |
590 FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));
591 set_64bit_val(qp_ctx, 144,
592 FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));
593 set_64bit_val(qp_ctx, 160,
594 FIELD_PREP(IRDMAQPC_PRIVEN, 1) |
595 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));
596 set_64bit_val(qp_ctx, 168,
597 FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));
598 set_64bit_val(qp_ctx, 176,
599 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
600 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
601 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
602
603 print_hex_dump_debug("PUDA: PUDA QP CONTEXT", DUMP_PREFIX_OFFSET, 16,
604 8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
605 }
606
607 /**
608 * irdma_puda_qp_wqe - setup wqe for qp create
609 * @dev: Device
610 * @qp: Resource qp
611 */
irdma_puda_qp_wqe(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)612 static enum irdma_status_code irdma_puda_qp_wqe(struct irdma_sc_dev *dev,
613 struct irdma_sc_qp *qp)
614 {
615 struct irdma_sc_cqp *cqp;
616 __le64 *wqe;
617 u64 hdr;
618 struct irdma_ccq_cqe_info compl_info;
619 enum irdma_status_code status = 0;
620
621 cqp = dev->cqp;
622 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
623 if (!wqe)
624 return IRDMA_ERR_RING_FULL;
625
626 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
627 set_64bit_val(wqe, 40, qp->shadow_area_pa);
628
629 hdr = qp->qp_uk.qp_id |
630 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
631 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |
632 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |
633 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |
634 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
635 dma_wmb(); /* make sure WQE is written before valid bit is set */
636
637 set_64bit_val(wqe, 24, hdr);
638
639 print_hex_dump_debug("PUDA: PUDA QP CREATE", DUMP_PREFIX_OFFSET, 16,
640 8, wqe, 40, false);
641 irdma_sc_cqp_post_sq(cqp);
642 status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_QP,
643 &compl_info);
644
645 return status;
646 }
647
648 /**
649 * irdma_puda_qp_create - create qp for resource
650 * @rsrc: resource to use for buffer
651 */
irdma_puda_qp_create(struct irdma_puda_rsrc * rsrc)652 static enum irdma_status_code irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
653 {
654 struct irdma_sc_qp *qp = &rsrc->qp;
655 struct irdma_qp_uk *ukqp = &qp->qp_uk;
656 enum irdma_status_code ret = 0;
657 u32 sq_size, rq_size;
658 struct irdma_dma_mem *mem;
659
660 sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;
661 rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;
662 rsrc->qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) + IRDMA_QP_CTX_SIZE),
663 IRDMA_HW_PAGE_SIZE);
664 rsrc->qpmem.va = dma_alloc_coherent(rsrc->dev->hw->device,
665 rsrc->qpmem.size, &rsrc->qpmem.pa,
666 GFP_KERNEL);
667 if (!rsrc->qpmem.va)
668 return IRDMA_ERR_NO_MEMORY;
669
670 mem = &rsrc->qpmem;
671 memset(mem->va, 0, rsrc->qpmem.size);
672 qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
673 qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
674 qp->pd = &rsrc->sc_pd;
675 qp->qp_uk.qp_type = IRDMA_QP_TYPE_UDA;
676 qp->dev = rsrc->dev;
677 qp->qp_uk.back_qp = rsrc;
678 qp->sq_pa = mem->pa;
679 qp->rq_pa = qp->sq_pa + sq_size;
680 qp->vsi = rsrc->vsi;
681 ukqp->sq_base = mem->va;
682 ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
683 ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
684 ukqp->uk_attrs = &qp->dev->hw_attrs.uk_attrs;
685 qp->shadow_area_pa = qp->rq_pa + rq_size;
686 qp->hw_host_ctx = ukqp->shadow_area + IRDMA_SHADOW_AREA_SIZE;
687 qp->hw_host_ctx_pa = qp->shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE << 3);
688 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
689 ukqp->qp_id = rsrc->qp_id;
690 ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
691 ukqp->rq_wrid_array = rsrc->rq_wrid_array;
692 ukqp->sq_size = rsrc->sq_size;
693 ukqp->rq_size = rsrc->rq_size;
694
695 IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
696 IRDMA_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
697 IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
698 ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
699
700 ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);
701 if (ret) {
702 dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
703 rsrc->qpmem.va, rsrc->qpmem.pa);
704 rsrc->qpmem.va = NULL;
705 return ret;
706 }
707
708 irdma_qp_add_qos(qp);
709 irdma_puda_qp_setctx(rsrc);
710
711 if (rsrc->dev->ceq_valid)
712 ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);
713 else
714 ret = irdma_puda_qp_wqe(rsrc->dev, qp);
715 if (ret) {
716 irdma_qp_rem_qos(qp);
717 rsrc->dev->ws_remove(qp->vsi, qp->user_pri);
718 dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
719 rsrc->qpmem.va, rsrc->qpmem.pa);
720 rsrc->qpmem.va = NULL;
721 }
722
723 return ret;
724 }
725
726 /**
727 * irdma_puda_cq_wqe - setup wqe for CQ create
728 * @dev: Device
729 * @cq: resource for cq
730 */
irdma_puda_cq_wqe(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)731 static enum irdma_status_code irdma_puda_cq_wqe(struct irdma_sc_dev *dev,
732 struct irdma_sc_cq *cq)
733 {
734 __le64 *wqe;
735 struct irdma_sc_cqp *cqp;
736 u64 hdr;
737 struct irdma_ccq_cqe_info compl_info;
738 enum irdma_status_code status = 0;
739
740 cqp = dev->cqp;
741 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
742 if (!wqe)
743 return IRDMA_ERR_RING_FULL;
744
745 set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
746 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
747 set_64bit_val(wqe, 16,
748 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
749 set_64bit_val(wqe, 32, cq->cq_pa);
750 set_64bit_val(wqe, 40, cq->shadow_area_pa);
751 set_64bit_val(wqe, 56,
752 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
753 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
754
755 hdr = cq->cq_uk.cq_id |
756 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
757 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |
758 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |
759 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |
760 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
761 dma_wmb(); /* make sure WQE is written before valid bit is set */
762
763 set_64bit_val(wqe, 24, hdr);
764
765 print_hex_dump_debug("PUDA: PUDA CREATE CQ", DUMP_PREFIX_OFFSET, 16,
766 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
767 irdma_sc_cqp_post_sq(dev->cqp);
768 status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_CQ,
769 &compl_info);
770 if (!status) {
771 struct irdma_sc_ceq *ceq = dev->ceq[0];
772
773 if (ceq && ceq->reg_cq)
774 status = irdma_sc_add_cq_ctx(ceq, cq);
775 }
776
777 return status;
778 }
779
780 /**
781 * irdma_puda_cq_create - create cq for resource
782 * @rsrc: resource for which cq to create
783 */
irdma_puda_cq_create(struct irdma_puda_rsrc * rsrc)784 static enum irdma_status_code irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
785 {
786 struct irdma_sc_dev *dev = rsrc->dev;
787 struct irdma_sc_cq *cq = &rsrc->cq;
788 enum irdma_status_code ret = 0;
789 u32 cqsize;
790 struct irdma_dma_mem *mem;
791 struct irdma_cq_init_info info = {};
792 struct irdma_cq_uk_init_info *init_info = &info.cq_uk_init_info;
793
794 cq->vsi = rsrc->vsi;
795 cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));
796 rsrc->cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),
797 IRDMA_CQ0_ALIGNMENT);
798 rsrc->cqmem.va = dma_alloc_coherent(dev->hw->device, rsrc->cqmem.size,
799 &rsrc->cqmem.pa, GFP_KERNEL);
800 if (!rsrc->cqmem.va)
801 return IRDMA_ERR_NO_MEMORY;
802
803 mem = &rsrc->cqmem;
804 info.dev = dev;
805 info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?
806 IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;
807 info.shadow_read_threshold = rsrc->cq_size >> 2;
808 info.cq_base_pa = mem->pa;
809 info.shadow_area_pa = mem->pa + cqsize;
810 init_info->cq_base = mem->va;
811 init_info->shadow_area = (__le64 *)((u8 *)mem->va + cqsize);
812 init_info->cq_size = rsrc->cq_size;
813 init_info->cq_id = rsrc->cq_id;
814 info.ceqe_mask = true;
815 info.ceq_id_valid = true;
816 info.vsi = rsrc->vsi;
817
818 ret = irdma_sc_cq_init(cq, &info);
819 if (ret)
820 goto error;
821
822 if (rsrc->dev->ceq_valid)
823 ret = irdma_cqp_cq_create_cmd(dev, cq);
824 else
825 ret = irdma_puda_cq_wqe(dev, cq);
826 error:
827 if (ret) {
828 dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
829 rsrc->cqmem.va, rsrc->cqmem.pa);
830 rsrc->cqmem.va = NULL;
831 }
832
833 return ret;
834 }
835
836 /**
837 * irdma_puda_free_qp - free qp for resource
838 * @rsrc: resource for which qp to free
839 */
irdma_puda_free_qp(struct irdma_puda_rsrc * rsrc)840 static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
841 {
842 enum irdma_status_code ret;
843 struct irdma_ccq_cqe_info compl_info;
844 struct irdma_sc_dev *dev = rsrc->dev;
845
846 if (rsrc->dev->ceq_valid) {
847 irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);
848 rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
849 return;
850 }
851
852 ret = irdma_sc_qp_destroy(&rsrc->qp, 0, false, true, true);
853 if (ret)
854 ibdev_dbg(to_ibdev(dev),
855 "PUDA: error puda qp destroy wqe, status = %d\n",
856 ret);
857 if (!ret) {
858 ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_QP,
859 &compl_info);
860 if (ret)
861 ibdev_dbg(to_ibdev(dev),
862 "PUDA: error puda qp destroy failed, status = %d\n",
863 ret);
864 }
865 rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
866 }
867
868 /**
869 * irdma_puda_free_cq - free cq for resource
870 * @rsrc: resource for which cq to free
871 */
irdma_puda_free_cq(struct irdma_puda_rsrc * rsrc)872 static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
873 {
874 enum irdma_status_code ret;
875 struct irdma_ccq_cqe_info compl_info;
876 struct irdma_sc_dev *dev = rsrc->dev;
877
878 if (rsrc->dev->ceq_valid) {
879 irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);
880 return;
881 }
882
883 ret = irdma_sc_cq_destroy(&rsrc->cq, 0, true);
884 if (ret)
885 ibdev_dbg(to_ibdev(dev), "PUDA: error ieq cq destroy\n");
886 if (!ret) {
887 ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_CQ,
888 &compl_info);
889 if (ret)
890 ibdev_dbg(to_ibdev(dev),
891 "PUDA: error ieq qp destroy done\n");
892 }
893 }
894
895 /**
896 * irdma_puda_dele_rsrc - delete all resources during close
897 * @vsi: VSI structure of device
898 * @type: type of resource to dele
899 * @reset: true if reset chip
900 */
irdma_puda_dele_rsrc(struct irdma_sc_vsi * vsi,enum puda_rsrc_type type,bool reset)901 void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
902 bool reset)
903 {
904 struct irdma_sc_dev *dev = vsi->dev;
905 struct irdma_puda_rsrc *rsrc;
906 struct irdma_puda_buf *buf = NULL;
907 struct irdma_puda_buf *nextbuf = NULL;
908 struct irdma_virt_mem *vmem;
909 struct irdma_sc_ceq *ceq;
910
911 ceq = vsi->dev->ceq[0];
912 switch (type) {
913 case IRDMA_PUDA_RSRC_TYPE_ILQ:
914 rsrc = vsi->ilq;
915 vmem = &vsi->ilq_mem;
916 vsi->ilq = NULL;
917 if (ceq && ceq->reg_cq)
918 irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
919 break;
920 case IRDMA_PUDA_RSRC_TYPE_IEQ:
921 rsrc = vsi->ieq;
922 vmem = &vsi->ieq_mem;
923 vsi->ieq = NULL;
924 if (ceq && ceq->reg_cq)
925 irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
926 break;
927 default:
928 ibdev_dbg(to_ibdev(dev), "PUDA: error resource type = 0x%x\n",
929 type);
930 return;
931 }
932
933 switch (rsrc->cmpl) {
934 case PUDA_HASH_CRC_COMPLETE:
935 irdma_free_hash_desc(rsrc->hash_desc);
936 fallthrough;
937 case PUDA_QP_CREATED:
938 irdma_qp_rem_qos(&rsrc->qp);
939
940 if (!reset)
941 irdma_puda_free_qp(rsrc);
942
943 dma_free_coherent(dev->hw->device, rsrc->qpmem.size,
944 rsrc->qpmem.va, rsrc->qpmem.pa);
945 rsrc->qpmem.va = NULL;
946 fallthrough;
947 case PUDA_CQ_CREATED:
948 if (!reset)
949 irdma_puda_free_cq(rsrc);
950
951 dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
952 rsrc->cqmem.va, rsrc->cqmem.pa);
953 rsrc->cqmem.va = NULL;
954 break;
955 default:
956 ibdev_dbg(to_ibdev(rsrc->dev), "PUDA: error no resources\n");
957 break;
958 }
959 /* Free all allocated puda buffers for both tx and rx */
960 buf = rsrc->alloclist;
961 while (buf) {
962 nextbuf = buf->next;
963 irdma_puda_dele_buf(dev, buf);
964 buf = nextbuf;
965 rsrc->alloc_buf_count--;
966 }
967
968 kfree(vmem->va);
969 }
970
971 /**
972 * irdma_puda_allocbufs - allocate buffers for resource
973 * @rsrc: resource for buffer allocation
974 * @count: number of buffers to create
975 */
irdma_puda_allocbufs(struct irdma_puda_rsrc * rsrc,u32 count)976 static enum irdma_status_code irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc,
977 u32 count)
978 {
979 u32 i;
980 struct irdma_puda_buf *buf;
981 struct irdma_puda_buf *nextbuf;
982
983 for (i = 0; i < count; i++) {
984 buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
985 if (!buf) {
986 rsrc->stats_buf_alloc_fail++;
987 return IRDMA_ERR_NO_MEMORY;
988 }
989 irdma_puda_ret_bufpool(rsrc, buf);
990 rsrc->alloc_buf_count++;
991 if (!rsrc->alloclist) {
992 rsrc->alloclist = buf;
993 } else {
994 nextbuf = rsrc->alloclist;
995 rsrc->alloclist = buf;
996 buf->next = nextbuf;
997 }
998 }
999
1000 rsrc->avail_buf_count = rsrc->alloc_buf_count;
1001
1002 return 0;
1003 }
1004
1005 /**
1006 * irdma_puda_create_rsrc - create resource (ilq or ieq)
1007 * @vsi: sc VSI struct
1008 * @info: resource information
1009 */
irdma_puda_create_rsrc(struct irdma_sc_vsi * vsi,struct irdma_puda_rsrc_info * info)1010 enum irdma_status_code irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
1011 struct irdma_puda_rsrc_info *info)
1012 {
1013 struct irdma_sc_dev *dev = vsi->dev;
1014 enum irdma_status_code ret = 0;
1015 struct irdma_puda_rsrc *rsrc;
1016 u32 pudasize;
1017 u32 sqwridsize, rqwridsize;
1018 struct irdma_virt_mem *vmem;
1019
1020 info->count = 1;
1021 pudasize = sizeof(struct irdma_puda_rsrc);
1022 sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
1023 rqwridsize = info->rq_size * 8;
1024 switch (info->type) {
1025 case IRDMA_PUDA_RSRC_TYPE_ILQ:
1026 vmem = &vsi->ilq_mem;
1027 break;
1028 case IRDMA_PUDA_RSRC_TYPE_IEQ:
1029 vmem = &vsi->ieq_mem;
1030 break;
1031 default:
1032 return IRDMA_NOT_SUPPORTED;
1033 }
1034 vmem->size = pudasize + sqwridsize + rqwridsize;
1035 vmem->va = kzalloc(vmem->size, GFP_KERNEL);
1036 if (!vmem->va)
1037 return IRDMA_ERR_NO_MEMORY;
1038
1039 rsrc = vmem->va;
1040 spin_lock_init(&rsrc->bufpool_lock);
1041 switch (info->type) {
1042 case IRDMA_PUDA_RSRC_TYPE_ILQ:
1043 vsi->ilq = vmem->va;
1044 vsi->ilq_count = info->count;
1045 rsrc->receive = info->receive;
1046 rsrc->xmit_complete = info->xmit_complete;
1047 break;
1048 case IRDMA_PUDA_RSRC_TYPE_IEQ:
1049 vsi->ieq_count = info->count;
1050 vsi->ieq = vmem->va;
1051 rsrc->receive = irdma_ieq_receive;
1052 rsrc->xmit_complete = irdma_ieq_tx_compl;
1053 break;
1054 default:
1055 return IRDMA_NOT_SUPPORTED;
1056 }
1057
1058 rsrc->type = info->type;
1059 rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
1060 ((u8 *)vmem->va + pudasize);
1061 rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
1062 /* Initialize all ieq lists */
1063 INIT_LIST_HEAD(&rsrc->bufpool);
1064 INIT_LIST_HEAD(&rsrc->txpend);
1065
1066 rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
1067 irdma_sc_pd_init(dev, &rsrc->sc_pd, info->pd_id, info->abi_ver);
1068 rsrc->qp_id = info->qp_id;
1069 rsrc->cq_id = info->cq_id;
1070 rsrc->sq_size = info->sq_size;
1071 rsrc->rq_size = info->rq_size;
1072 rsrc->cq_size = info->rq_size + info->sq_size;
1073 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1074 if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1075 rsrc->cq_size += info->rq_size;
1076 }
1077 rsrc->buf_size = info->buf_size;
1078 rsrc->dev = dev;
1079 rsrc->vsi = vsi;
1080 rsrc->stats_idx = info->stats_idx;
1081 rsrc->stats_idx_valid = info->stats_idx_valid;
1082
1083 ret = irdma_puda_cq_create(rsrc);
1084 if (!ret) {
1085 rsrc->cmpl = PUDA_CQ_CREATED;
1086 ret = irdma_puda_qp_create(rsrc);
1087 }
1088 if (ret) {
1089 ibdev_dbg(to_ibdev(dev),
1090 "PUDA: error qp_create type=%d, status=%d\n",
1091 rsrc->type, ret);
1092 goto error;
1093 }
1094 rsrc->cmpl = PUDA_QP_CREATED;
1095
1096 ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
1097 if (ret) {
1098 ibdev_dbg(to_ibdev(dev), "PUDA: error alloc_buf\n");
1099 goto error;
1100 }
1101
1102 rsrc->rxq_invalid_cnt = info->rq_size;
1103 ret = irdma_puda_replenish_rq(rsrc, true);
1104 if (ret)
1105 goto error;
1106
1107 if (info->type == IRDMA_PUDA_RSRC_TYPE_IEQ) {
1108 if (!irdma_init_hash_desc(&rsrc->hash_desc)) {
1109 rsrc->check_crc = true;
1110 rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;
1111 ret = 0;
1112 }
1113 }
1114
1115 irdma_sc_ccq_arm(&rsrc->cq);
1116 return ret;
1117
1118 error:
1119 irdma_puda_dele_rsrc(vsi, info->type, false);
1120
1121 return ret;
1122 }
1123
1124 /**
1125 * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq
1126 * @qp: ilq's qp resource
1127 * @buf: puda buffer for rcv q
1128 * @wqe_idx: wqe index of completed rcvbuf
1129 */
irdma_ilq_putback_rcvbuf(struct irdma_sc_qp * qp,struct irdma_puda_buf * buf,u32 wqe_idx)1130 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
1131 struct irdma_puda_buf *buf, u32 wqe_idx)
1132 {
1133 __le64 *wqe;
1134 u64 offset8, offset24;
1135
1136 /* Synch buffer for use by device */
1137 dma_sync_single_for_device(qp->dev->hw->device, buf->mem.pa,
1138 buf->mem.size, DMA_BIDIRECTIONAL);
1139 wqe = qp->qp_uk.rq_base[wqe_idx].elem;
1140 get_64bit_val(wqe, 24, &offset24);
1141 if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1142 get_64bit_val(wqe, 8, &offset8);
1143 if (offset24)
1144 offset8 &= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);
1145 else
1146 offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);
1147 set_64bit_val(wqe, 8, offset8);
1148 dma_wmb(); /* make sure WQE is written before valid bit is set */
1149 }
1150 if (offset24)
1151 offset24 = 0;
1152 else
1153 offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);
1154
1155 set_64bit_val(wqe, 24, offset24);
1156 }
1157
1158 /**
1159 * irdma_ieq_get_fpdu_len - get length of fpdu with or without marker
1160 * @pfpdu: pointer to fpdu
1161 * @datap: pointer to data in the buffer
1162 * @rcv_seq: seqnum of the data buffer
1163 */
irdma_ieq_get_fpdu_len(struct irdma_pfpdu * pfpdu,u8 * datap,u32 rcv_seq)1164 static u16 irdma_ieq_get_fpdu_len(struct irdma_pfpdu *pfpdu, u8 *datap,
1165 u32 rcv_seq)
1166 {
1167 u32 marker_seq, end_seq, blk_start;
1168 u8 marker_len = pfpdu->marker_len;
1169 u16 total_len = 0;
1170 u16 fpdu_len;
1171
1172 blk_start = (pfpdu->rcv_start_seq - rcv_seq) & (IRDMA_MRK_BLK_SZ - 1);
1173 if (!blk_start) {
1174 total_len = marker_len;
1175 marker_seq = rcv_seq + IRDMA_MRK_BLK_SZ;
1176 if (marker_len && *(u32 *)datap)
1177 return 0;
1178 } else {
1179 marker_seq = rcv_seq + blk_start;
1180 }
1181
1182 datap += total_len;
1183 fpdu_len = ntohs(*(__be16 *)datap);
1184 fpdu_len += IRDMA_IEQ_MPA_FRAMING;
1185 fpdu_len = (fpdu_len + 3) & 0xfffc;
1186
1187 if (fpdu_len > pfpdu->max_fpdu_data)
1188 return 0;
1189
1190 total_len += fpdu_len;
1191 end_seq = rcv_seq + total_len;
1192 while ((int)(marker_seq - end_seq) < 0) {
1193 total_len += marker_len;
1194 end_seq += marker_len;
1195 marker_seq += IRDMA_MRK_BLK_SZ;
1196 }
1197
1198 return total_len;
1199 }
1200
1201 /**
1202 * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
1203 * @buf: rcv buffer with partial
1204 * @txbuf: tx buffer for sending back
1205 * @buf_offset: rcv buffer offset to copy from
1206 * @txbuf_offset: at offset in tx buf to copy
1207 * @len: length of data to copy
1208 */
irdma_ieq_copy_to_txbuf(struct irdma_puda_buf * buf,struct irdma_puda_buf * txbuf,u16 buf_offset,u32 txbuf_offset,u32 len)1209 static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,
1210 struct irdma_puda_buf *txbuf,
1211 u16 buf_offset, u32 txbuf_offset, u32 len)
1212 {
1213 void *mem1 = (u8 *)buf->mem.va + buf_offset;
1214 void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
1215
1216 memcpy(mem2, mem1, len);
1217 }
1218
1219 /**
1220 * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling
1221 * @buf: reeive buffer with partial
1222 * @txbuf: buffer to prepare
1223 */
irdma_ieq_setup_tx_buf(struct irdma_puda_buf * buf,struct irdma_puda_buf * txbuf)1224 static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,
1225 struct irdma_puda_buf *txbuf)
1226 {
1227 txbuf->tcphlen = buf->tcphlen;
1228 txbuf->ipv4 = buf->ipv4;
1229
1230 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1231 txbuf->hdrlen = txbuf->tcphlen;
1232 irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,
1233 txbuf->hdrlen);
1234 } else {
1235 txbuf->maclen = buf->maclen;
1236 txbuf->hdrlen = buf->hdrlen;
1237 irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
1238 }
1239 }
1240
1241 /**
1242 * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range
1243 * @buf: receive exception buffer
1244 * @fps: first partial sequence number
1245 */
irdma_ieq_check_first_buf(struct irdma_puda_buf * buf,u32 fps)1246 static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)
1247 {
1248 u32 offset;
1249
1250 if (buf->seqnum < fps) {
1251 offset = fps - buf->seqnum;
1252 if (offset > buf->datalen)
1253 return;
1254 buf->data += offset;
1255 buf->datalen -= (u16)offset;
1256 buf->seqnum = fps;
1257 }
1258 }
1259
1260 /**
1261 * irdma_ieq_compl_pfpdu - write txbuf with full fpdu
1262 * @ieq: ieq resource
1263 * @rxlist: ieq's received buffer list
1264 * @pbufl: temporary list for buffers for fpddu
1265 * @txbuf: tx buffer for fpdu
1266 * @fpdu_len: total length of fpdu
1267 */
irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc * ieq,struct list_head * rxlist,struct list_head * pbufl,struct irdma_puda_buf * txbuf,u16 fpdu_len)1268 static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
1269 struct list_head *rxlist,
1270 struct list_head *pbufl,
1271 struct irdma_puda_buf *txbuf, u16 fpdu_len)
1272 {
1273 struct irdma_puda_buf *buf;
1274 u32 nextseqnum;
1275 u16 txoffset, bufoffset;
1276
1277 buf = irdma_puda_get_listbuf(pbufl);
1278 if (!buf)
1279 return;
1280
1281 nextseqnum = buf->seqnum + fpdu_len;
1282 irdma_ieq_setup_tx_buf(buf, txbuf);
1283 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1284 txoffset = txbuf->hdrlen;
1285 txbuf->totallen = txbuf->hdrlen + fpdu_len;
1286 txbuf->data = (u8 *)txbuf->mem.va + txoffset;
1287 } else {
1288 txoffset = buf->hdrlen;
1289 txbuf->totallen = buf->hdrlen + fpdu_len;
1290 txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
1291 }
1292 bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1293
1294 do {
1295 if (buf->datalen >= fpdu_len) {
1296 /* copied full fpdu */
1297 irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1298 fpdu_len);
1299 buf->datalen -= fpdu_len;
1300 buf->data += fpdu_len;
1301 buf->seqnum = nextseqnum;
1302 break;
1303 }
1304 /* copy partial fpdu */
1305 irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1306 buf->datalen);
1307 txoffset += buf->datalen;
1308 fpdu_len -= buf->datalen;
1309 irdma_puda_ret_bufpool(ieq, buf);
1310 buf = irdma_puda_get_listbuf(pbufl);
1311 if (!buf)
1312 return;
1313
1314 bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1315 } while (1);
1316
1317 /* last buffer on the list*/
1318 if (buf->datalen)
1319 list_add(&buf->list, rxlist);
1320 else
1321 irdma_puda_ret_bufpool(ieq, buf);
1322 }
1323
1324 /**
1325 * irdma_ieq_create_pbufl - create buffer list for single fpdu
1326 * @pfpdu: pointer to fpdu
1327 * @rxlist: resource list for receive ieq buffes
1328 * @pbufl: temp. list for buffers for fpddu
1329 * @buf: first receive buffer
1330 * @fpdu_len: total length of fpdu
1331 */
1332 static enum irdma_status_code
irdma_ieq_create_pbufl(struct irdma_pfpdu * pfpdu,struct list_head * rxlist,struct list_head * pbufl,struct irdma_puda_buf * buf,u16 fpdu_len)1333 irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu, struct list_head *rxlist,
1334 struct list_head *pbufl, struct irdma_puda_buf *buf,
1335 u16 fpdu_len)
1336 {
1337 enum irdma_status_code status = 0;
1338 struct irdma_puda_buf *nextbuf;
1339 u32 nextseqnum;
1340 u16 plen = fpdu_len - buf->datalen;
1341 bool done = false;
1342
1343 nextseqnum = buf->seqnum + buf->datalen;
1344 do {
1345 nextbuf = irdma_puda_get_listbuf(rxlist);
1346 if (!nextbuf) {
1347 status = IRDMA_ERR_list_empty;
1348 break;
1349 }
1350 list_add_tail(&nextbuf->list, pbufl);
1351 if (nextbuf->seqnum != nextseqnum) {
1352 pfpdu->bad_seq_num++;
1353 status = IRDMA_ERR_SEQ_NUM;
1354 break;
1355 }
1356 if (nextbuf->datalen >= plen) {
1357 done = true;
1358 } else {
1359 plen -= nextbuf->datalen;
1360 nextseqnum = nextbuf->seqnum + nextbuf->datalen;
1361 }
1362
1363 } while (!done);
1364
1365 return status;
1366 }
1367
1368 /**
1369 * irdma_ieq_handle_partial - process partial fpdu buffer
1370 * @ieq: ieq resource
1371 * @pfpdu: partial management per user qp
1372 * @buf: receive buffer
1373 * @fpdu_len: fpdu len in the buffer
1374 */
1375 static enum irdma_status_code
irdma_ieq_handle_partial(struct irdma_puda_rsrc * ieq,struct irdma_pfpdu * pfpdu,struct irdma_puda_buf * buf,u16 fpdu_len)1376 irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq, struct irdma_pfpdu *pfpdu,
1377 struct irdma_puda_buf *buf, u16 fpdu_len)
1378 {
1379 enum irdma_status_code status = 0;
1380 u8 *crcptr;
1381 u32 mpacrc;
1382 u32 seqnum = buf->seqnum;
1383 struct list_head pbufl; /* partial buffer list */
1384 struct irdma_puda_buf *txbuf = NULL;
1385 struct list_head *rxlist = &pfpdu->rxlist;
1386
1387 ieq->partials_handled++;
1388
1389 INIT_LIST_HEAD(&pbufl);
1390 list_add(&buf->list, &pbufl);
1391
1392 status = irdma_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
1393 if (status)
1394 goto error;
1395
1396 txbuf = irdma_puda_get_bufpool(ieq);
1397 if (!txbuf) {
1398 pfpdu->no_tx_bufs++;
1399 status = IRDMA_ERR_NO_TXBUFS;
1400 goto error;
1401 }
1402
1403 irdma_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
1404 irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
1405
1406 crcptr = txbuf->data + fpdu_len - 4;
1407 mpacrc = *(u32 *)crcptr;
1408 if (ieq->check_crc) {
1409 status = irdma_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
1410 (fpdu_len - 4), mpacrc);
1411 if (status) {
1412 ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error bad crc\n");
1413 goto error;
1414 }
1415 }
1416
1417 print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1418 txbuf->mem.va, txbuf->totallen, false);
1419 if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1420 txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1421 txbuf->do_lpb = true;
1422 irdma_puda_send_buf(ieq, txbuf);
1423 pfpdu->rcv_nxt = seqnum + fpdu_len;
1424 return status;
1425
1426 error:
1427 while (!list_empty(&pbufl)) {
1428 buf = list_last_entry(&pbufl, struct irdma_puda_buf, list);
1429 list_move(&buf->list, rxlist);
1430 }
1431 if (txbuf)
1432 irdma_puda_ret_bufpool(ieq, txbuf);
1433
1434 return status;
1435 }
1436
1437 /**
1438 * irdma_ieq_process_buf - process buffer rcvd for ieq
1439 * @ieq: ieq resource
1440 * @pfpdu: partial management per user qp
1441 * @buf: receive buffer
1442 */
irdma_ieq_process_buf(struct irdma_puda_rsrc * ieq,struct irdma_pfpdu * pfpdu,struct irdma_puda_buf * buf)1443 static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
1444 struct irdma_pfpdu *pfpdu,
1445 struct irdma_puda_buf *buf)
1446 {
1447 u16 fpdu_len = 0;
1448 u16 datalen = buf->datalen;
1449 u8 *datap = buf->data;
1450 u8 *crcptr;
1451 u16 ioffset = 0;
1452 u32 mpacrc;
1453 u32 seqnum = buf->seqnum;
1454 u16 len = 0;
1455 u16 full = 0;
1456 bool partial = false;
1457 struct irdma_puda_buf *txbuf;
1458 struct list_head *rxlist = &pfpdu->rxlist;
1459 enum irdma_status_code ret = 0;
1460
1461 ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
1462 while (datalen) {
1463 fpdu_len = irdma_ieq_get_fpdu_len(pfpdu, datap, buf->seqnum);
1464 if (!fpdu_len) {
1465 ibdev_dbg(to_ibdev(ieq->dev),
1466 "IEQ: error bad fpdu len\n");
1467 list_add(&buf->list, rxlist);
1468 return IRDMA_ERR_MPA_CRC;
1469 }
1470
1471 if (datalen < fpdu_len) {
1472 partial = true;
1473 break;
1474 }
1475 crcptr = datap + fpdu_len - 4;
1476 mpacrc = *(u32 *)crcptr;
1477 if (ieq->check_crc)
1478 ret = irdma_ieq_check_mpacrc(ieq->hash_desc, datap,
1479 fpdu_len - 4, mpacrc);
1480 if (ret) {
1481 list_add(&buf->list, rxlist);
1482 ibdev_dbg(to_ibdev(ieq->dev),
1483 "ERR: IRDMA_ERR_MPA_CRC\n");
1484 return IRDMA_ERR_MPA_CRC;
1485 }
1486 full++;
1487 pfpdu->fpdu_processed++;
1488 ieq->fpdu_processed++;
1489 datap += fpdu_len;
1490 len += fpdu_len;
1491 datalen -= fpdu_len;
1492 }
1493 if (full) {
1494 /* copy full pdu's in the txbuf and send them out */
1495 txbuf = irdma_puda_get_bufpool(ieq);
1496 if (!txbuf) {
1497 pfpdu->no_tx_bufs++;
1498 list_add(&buf->list, rxlist);
1499 return IRDMA_ERR_NO_TXBUFS;
1500 }
1501 /* modify txbuf's buffer header */
1502 irdma_ieq_setup_tx_buf(buf, txbuf);
1503 /* copy full fpdu's to new buffer */
1504 if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1505 irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1506 txbuf->hdrlen, len);
1507 txbuf->totallen = txbuf->hdrlen + len;
1508 txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1509 } else {
1510 irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1511 buf->hdrlen, len);
1512 txbuf->totallen = buf->hdrlen + len;
1513 }
1514 irdma_ieq_update_tcpip_info(txbuf, len, buf->seqnum);
1515 print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET,
1516 16, 8, txbuf->mem.va, txbuf->totallen,
1517 false);
1518 txbuf->do_lpb = true;
1519 irdma_puda_send_buf(ieq, txbuf);
1520
1521 if (!datalen) {
1522 pfpdu->rcv_nxt = buf->seqnum + len;
1523 irdma_puda_ret_bufpool(ieq, buf);
1524 return 0;
1525 }
1526 buf->data = datap;
1527 buf->seqnum = seqnum + len;
1528 buf->datalen = datalen;
1529 pfpdu->rcv_nxt = buf->seqnum;
1530 }
1531 if (partial)
1532 return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
1533
1534 return 0;
1535 }
1536
1537 /**
1538 * irdma_ieq_process_fpdus - process fpdu's buffers on its list
1539 * @qp: qp for which partial fpdus
1540 * @ieq: ieq resource
1541 */
irdma_ieq_process_fpdus(struct irdma_sc_qp * qp,struct irdma_puda_rsrc * ieq)1542 void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
1543 struct irdma_puda_rsrc *ieq)
1544 {
1545 struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1546 struct list_head *rxlist = &pfpdu->rxlist;
1547 struct irdma_puda_buf *buf;
1548 enum irdma_status_code status;
1549
1550 do {
1551 if (list_empty(rxlist))
1552 break;
1553 buf = irdma_puda_get_listbuf(rxlist);
1554 if (!buf) {
1555 ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error no buf\n");
1556 break;
1557 }
1558 if (buf->seqnum != pfpdu->rcv_nxt) {
1559 /* This could be out of order or missing packet */
1560 pfpdu->out_of_order++;
1561 list_add(&buf->list, rxlist);
1562 break;
1563 }
1564 /* keep processing buffers from the head of the list */
1565 status = irdma_ieq_process_buf(ieq, pfpdu, buf);
1566 if (status == IRDMA_ERR_MPA_CRC) {
1567 pfpdu->mpa_crc_err = true;
1568 while (!list_empty(rxlist)) {
1569 buf = irdma_puda_get_listbuf(rxlist);
1570 irdma_puda_ret_bufpool(ieq, buf);
1571 pfpdu->crc_err++;
1572 ieq->crc_err++;
1573 }
1574 /* create CQP for AE */
1575 irdma_ieq_mpa_crc_ae(ieq->dev, qp);
1576 }
1577 } while (!status);
1578 }
1579
1580 /**
1581 * irdma_ieq_create_ah - create an address handle for IEQ
1582 * @qp: qp pointer
1583 * @buf: buf received on IEQ used to create AH
1584 */
irdma_ieq_create_ah(struct irdma_sc_qp * qp,struct irdma_puda_buf * buf)1585 static enum irdma_status_code irdma_ieq_create_ah(struct irdma_sc_qp *qp,
1586 struct irdma_puda_buf *buf)
1587 {
1588 struct irdma_ah_info ah_info = {};
1589
1590 qp->pfpdu.ah_buf = buf;
1591 irdma_puda_ieq_get_ah_info(qp, &ah_info);
1592 return irdma_puda_create_ah(qp->vsi->dev, &ah_info, false,
1593 IRDMA_PUDA_RSRC_TYPE_IEQ, qp,
1594 &qp->pfpdu.ah);
1595 }
1596
1597 /**
1598 * irdma_ieq_handle_exception - handle qp's exception
1599 * @ieq: ieq resource
1600 * @qp: qp receiving excpetion
1601 * @buf: receive buffer
1602 */
irdma_ieq_handle_exception(struct irdma_puda_rsrc * ieq,struct irdma_sc_qp * qp,struct irdma_puda_buf * buf)1603 static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
1604 struct irdma_sc_qp *qp,
1605 struct irdma_puda_buf *buf)
1606 {
1607 struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1608 u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
1609 u32 rcv_wnd = hw_host_ctx[23];
1610 /* first partial seq # in q2 */
1611 u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
1612 struct list_head *rxlist = &pfpdu->rxlist;
1613 unsigned long flags = 0;
1614 u8 hw_rev = qp->dev->hw_attrs.uk_attrs.hw_rev;
1615
1616 print_hex_dump_debug("IEQ: IEQ RX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1617 buf->mem.va, buf->totallen, false);
1618
1619 spin_lock_irqsave(&pfpdu->lock, flags);
1620 pfpdu->total_ieq_bufs++;
1621 if (pfpdu->mpa_crc_err) {
1622 pfpdu->crc_err++;
1623 goto error;
1624 }
1625 if (pfpdu->mode && fps != pfpdu->fps) {
1626 /* clean up qp as it is new partial sequence */
1627 irdma_ieq_cleanup_qp(ieq, qp);
1628 ibdev_dbg(to_ibdev(ieq->dev), "IEQ: restarting new partial\n");
1629 pfpdu->mode = false;
1630 }
1631
1632 if (!pfpdu->mode) {
1633 print_hex_dump_debug("IEQ: Q2 BUFFER", DUMP_PREFIX_OFFSET, 16,
1634 8, (u64 *)qp->q2_buf, 128, false);
1635 /* First_Partial_Sequence_Number check */
1636 pfpdu->rcv_nxt = fps;
1637 pfpdu->fps = fps;
1638 pfpdu->mode = true;
1639 pfpdu->max_fpdu_data = (buf->ipv4) ?
1640 (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV4) :
1641 (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV6);
1642 pfpdu->pmode_count++;
1643 ieq->pmode_count++;
1644 INIT_LIST_HEAD(rxlist);
1645 irdma_ieq_check_first_buf(buf, fps);
1646 }
1647
1648 if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
1649 pfpdu->bad_seq_num++;
1650 ieq->bad_seq_num++;
1651 goto error;
1652 }
1653
1654 if (!list_empty(rxlist)) {
1655 if (buf->seqnum != pfpdu->nextseqnum) {
1656 irdma_send_ieq_ack(qp);
1657 /* throw away out-of-order, duplicates*/
1658 goto error;
1659 }
1660 }
1661 /* Insert buf before head */
1662 list_add_tail(&buf->list, rxlist);
1663 pfpdu->nextseqnum = buf->seqnum + buf->datalen;
1664 pfpdu->lastrcv_buf = buf;
1665 if (hw_rev >= IRDMA_GEN_2 && !pfpdu->ah) {
1666 irdma_ieq_create_ah(qp, buf);
1667 if (!pfpdu->ah)
1668 goto error;
1669 goto exit;
1670 }
1671 if (hw_rev == IRDMA_GEN_1)
1672 irdma_ieq_process_fpdus(qp, ieq);
1673 else if (pfpdu->ah && pfpdu->ah->ah_info.ah_valid)
1674 irdma_ieq_process_fpdus(qp, ieq);
1675 exit:
1676 spin_unlock_irqrestore(&pfpdu->lock, flags);
1677
1678 return;
1679
1680 error:
1681 irdma_puda_ret_bufpool(ieq, buf);
1682 spin_unlock_irqrestore(&pfpdu->lock, flags);
1683 }
1684
1685 /**
1686 * irdma_ieq_receive - received exception buffer
1687 * @vsi: VSI of device
1688 * @buf: exception buffer received
1689 */
irdma_ieq_receive(struct irdma_sc_vsi * vsi,struct irdma_puda_buf * buf)1690 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
1691 struct irdma_puda_buf *buf)
1692 {
1693 struct irdma_puda_rsrc *ieq = vsi->ieq;
1694 struct irdma_sc_qp *qp = NULL;
1695 u32 wqe_idx = ieq->compl_rxwqe_idx;
1696
1697 qp = irdma_ieq_get_qp(vsi->dev, buf);
1698 if (!qp) {
1699 ieq->stats_bad_qp_id++;
1700 irdma_puda_ret_bufpool(ieq, buf);
1701 } else {
1702 irdma_ieq_handle_exception(ieq, qp, buf);
1703 }
1704 /*
1705 * ieq->rx_wqe_idx is used by irdma_puda_replenish_rq()
1706 * on which wqe_idx to start replenish rq
1707 */
1708 if (!ieq->rxq_invalid_cnt)
1709 ieq->rx_wqe_idx = wqe_idx;
1710 ieq->rxq_invalid_cnt++;
1711 }
1712
1713 /**
1714 * irdma_ieq_tx_compl - put back after sending completed exception buffer
1715 * @vsi: sc VSI struct
1716 * @sqwrid: pointer to puda buffer
1717 */
irdma_ieq_tx_compl(struct irdma_sc_vsi * vsi,void * sqwrid)1718 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)
1719 {
1720 struct irdma_puda_rsrc *ieq = vsi->ieq;
1721 struct irdma_puda_buf *buf = sqwrid;
1722
1723 irdma_puda_ret_bufpool(ieq, buf);
1724 }
1725
1726 /**
1727 * irdma_ieq_cleanup_qp - qp is being destroyed
1728 * @ieq: ieq resource
1729 * @qp: all pending fpdu buffers
1730 */
irdma_ieq_cleanup_qp(struct irdma_puda_rsrc * ieq,struct irdma_sc_qp * qp)1731 void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)
1732 {
1733 struct irdma_puda_buf *buf;
1734 struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1735 struct list_head *rxlist = &pfpdu->rxlist;
1736
1737 if (qp->pfpdu.ah) {
1738 irdma_puda_free_ah(ieq->dev, qp->pfpdu.ah);
1739 qp->pfpdu.ah = NULL;
1740 qp->pfpdu.ah_buf = NULL;
1741 }
1742
1743 if (!pfpdu->mode)
1744 return;
1745
1746 while (!list_empty(rxlist)) {
1747 buf = irdma_puda_get_listbuf(rxlist);
1748 irdma_puda_ret_bufpool(ieq, buf);
1749 }
1750 }
1751