1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include <linux/etherdevice.h>
4
5 #include "osdep.h"
6 #include "hmc.h"
7 #include "defs.h"
8 #include "type.h"
9 #include "ws.h"
10 #include "protos.h"
11
12 /**
13 * irdma_get_qp_from_list - get next qp from a list
14 * @head: Listhead of qp's
15 * @qp: current qp
16 */
irdma_get_qp_from_list(struct list_head * head,struct irdma_sc_qp * qp)17 struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
18 struct irdma_sc_qp *qp)
19 {
20 struct list_head *lastentry;
21 struct list_head *entry = NULL;
22
23 if (list_empty(head))
24 return NULL;
25
26 if (!qp) {
27 entry = head->next;
28 } else {
29 lastentry = &qp->list;
30 entry = lastentry->next;
31 if (entry == head)
32 return NULL;
33 }
34
35 return container_of(entry, struct irdma_sc_qp, list);
36 }
37
38 /**
39 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI
40 * @vsi: the VSI struct pointer
41 * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND
42 */
irdma_sc_suspend_resume_qps(struct irdma_sc_vsi * vsi,u8 op)43 void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op)
44 {
45 struct irdma_sc_qp *qp = NULL;
46 u8 i;
47
48 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
49 mutex_lock(&vsi->qos[i].qos_mutex);
50 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
51 while (qp) {
52 if (op == IRDMA_OP_RESUME) {
53 if (!qp->dev->ws_add(vsi, i)) {
54 qp->qs_handle =
55 vsi->qos[qp->user_pri].qs_handle;
56 irdma_cqp_qp_suspend_resume(qp, op);
57 } else {
58 irdma_cqp_qp_suspend_resume(qp, op);
59 irdma_modify_qp_to_err(qp);
60 }
61 } else if (op == IRDMA_OP_SUSPEND) {
62 /* issue cqp suspend command */
63 if (!irdma_cqp_qp_suspend_resume(qp, op))
64 atomic_inc(&vsi->qp_suspend_reqs);
65 }
66 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
67 }
68 mutex_unlock(&vsi->qos[i].qos_mutex);
69 }
70 }
71
irdma_set_qos_info(struct irdma_sc_vsi * vsi,struct irdma_l2params * l2p)72 static void irdma_set_qos_info(struct irdma_sc_vsi *vsi,
73 struct irdma_l2params *l2p)
74 {
75 u8 i;
76
77 vsi->qos_rel_bw = l2p->vsi_rel_bw;
78 vsi->qos_prio_type = l2p->vsi_prio_type;
79 vsi->dscp_mode = l2p->dscp_mode;
80 if (l2p->dscp_mode) {
81 memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map));
82 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
83 l2p->up2tc[i] = i;
84 }
85 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
86 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
87 vsi->qos[i].qs_handle = l2p->qs_handle_list[i];
88 vsi->qos[i].traffic_class = l2p->up2tc[i];
89 vsi->qos[i].rel_bw =
90 l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;
91 vsi->qos[i].prio_type =
92 l2p->tc_info[vsi->qos[i].traffic_class].prio_type;
93 vsi->qos[i].valid = false;
94 }
95 }
96
97 /**
98 * irdma_change_l2params - given the new l2 parameters, change all qp
99 * @vsi: RDMA VSI pointer
100 * @l2params: New parameters from l2
101 */
irdma_change_l2params(struct irdma_sc_vsi * vsi,struct irdma_l2params * l2params)102 void irdma_change_l2params(struct irdma_sc_vsi *vsi,
103 struct irdma_l2params *l2params)
104 {
105 if (l2params->mtu_changed) {
106 vsi->mtu = l2params->mtu;
107 if (vsi->ieq)
108 irdma_reinitialize_ieq(vsi);
109 }
110
111 if (!l2params->tc_changed)
112 return;
113
114 vsi->tc_change_pending = false;
115 irdma_set_qos_info(vsi, l2params);
116 irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);
117 }
118
119 /**
120 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
121 * @qp: qp to be removed from qos
122 */
irdma_qp_rem_qos(struct irdma_sc_qp * qp)123 void irdma_qp_rem_qos(struct irdma_sc_qp *qp)
124 {
125 struct irdma_sc_vsi *vsi = qp->vsi;
126
127 ibdev_dbg(to_ibdev(qp->dev),
128 "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
129 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
130 qp->on_qoslist);
131 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
132 if (qp->on_qoslist) {
133 qp->on_qoslist = false;
134 list_del(&qp->list);
135 }
136 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
137 }
138
139 /**
140 * irdma_qp_add_qos - called during setctx for qp to be added to qos
141 * @qp: qp to be added to qos
142 */
irdma_qp_add_qos(struct irdma_sc_qp * qp)143 void irdma_qp_add_qos(struct irdma_sc_qp *qp)
144 {
145 struct irdma_sc_vsi *vsi = qp->vsi;
146
147 ibdev_dbg(to_ibdev(qp->dev),
148 "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
149 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
150 qp->on_qoslist);
151 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
152 if (!qp->on_qoslist) {
153 list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
154 qp->on_qoslist = true;
155 qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
156 }
157 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
158 }
159
160 /**
161 * irdma_sc_pd_init - initialize sc pd struct
162 * @dev: sc device struct
163 * @pd: sc pd ptr
164 * @pd_id: pd_id for allocated pd
165 * @abi_ver: User/Kernel ABI version
166 */
irdma_sc_pd_init(struct irdma_sc_dev * dev,struct irdma_sc_pd * pd,u32 pd_id,int abi_ver)167 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
168 int abi_ver)
169 {
170 pd->pd_id = pd_id;
171 pd->abi_ver = abi_ver;
172 pd->dev = dev;
173 }
174
175 /**
176 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
177 * @cqp: struct for cqp hw
178 * @info: arp entry information
179 * @scratch: u64 saved to be used during cqp completion
180 * @post_sq: flag for cqp db to ring
181 */
irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp * cqp,struct irdma_add_arp_cache_entry_info * info,u64 scratch,bool post_sq)182 static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,
183 struct irdma_add_arp_cache_entry_info *info,
184 u64 scratch, bool post_sq)
185 {
186 __le64 *wqe;
187 u64 hdr;
188
189 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
190 if (!wqe)
191 return -ENOMEM;
192 set_64bit_val(wqe, 8, info->reach_max);
193 set_64bit_val(wqe, 16, ether_addr_to_u64(info->mac_addr));
194
195 hdr = info->arp_index |
196 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
197 FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) |
198 FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, 1) |
199 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
200 dma_wmb(); /* make sure WQE is written before valid bit is set */
201
202 set_64bit_val(wqe, 24, hdr);
203
204 print_hex_dump_debug("WQE: ARP_CACHE_ENTRY WQE", DUMP_PREFIX_OFFSET,
205 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
206 if (post_sq)
207 irdma_sc_cqp_post_sq(cqp);
208
209 return 0;
210 }
211
212 /**
213 * irdma_sc_del_arp_cache_entry - dele arp cache entry
214 * @cqp: struct for cqp hw
215 * @scratch: u64 saved to be used during cqp completion
216 * @arp_index: arp index to delete arp entry
217 * @post_sq: flag for cqp db to ring
218 */
irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp * cqp,u64 scratch,u16 arp_index,bool post_sq)219 static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,
220 u16 arp_index, bool post_sq)
221 {
222 __le64 *wqe;
223 u64 hdr;
224
225 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
226 if (!wqe)
227 return -ENOMEM;
228
229 hdr = arp_index |
230 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
231 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
232 dma_wmb(); /* make sure WQE is written before valid bit is set */
233
234 set_64bit_val(wqe, 24, hdr);
235
236 print_hex_dump_debug("WQE: ARP_CACHE_DEL_ENTRY WQE",
237 DUMP_PREFIX_OFFSET, 16, 8, wqe,
238 IRDMA_CQP_WQE_SIZE * 8, false);
239 if (post_sq)
240 irdma_sc_cqp_post_sq(cqp);
241
242 return 0;
243 }
244
245 /**
246 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
247 * @cqp: struct for cqp hw
248 * @info: info for apbvt entry to add or delete
249 * @scratch: u64 saved to be used during cqp completion
250 * @post_sq: flag for cqp db to ring
251 */
irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp * cqp,struct irdma_apbvt_info * info,u64 scratch,bool post_sq)252 static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,
253 struct irdma_apbvt_info *info,
254 u64 scratch, bool post_sq)
255 {
256 __le64 *wqe;
257 u64 hdr;
258
259 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
260 if (!wqe)
261 return -ENOMEM;
262
263 set_64bit_val(wqe, 16, info->port);
264
265 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) |
266 FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) |
267 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
268 dma_wmb(); /* make sure WQE is written before valid bit is set */
269
270 set_64bit_val(wqe, 24, hdr);
271
272 print_hex_dump_debug("WQE: MANAGE_APBVT WQE", DUMP_PREFIX_OFFSET, 16,
273 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
274 if (post_sq)
275 irdma_sc_cqp_post_sq(cqp);
276
277 return 0;
278 }
279
280 /**
281 * irdma_sc_manage_qhash_table_entry - manage quad hash entries
282 * @cqp: struct for cqp hw
283 * @info: info for quad hash to manage
284 * @scratch: u64 saved to be used during cqp completion
285 * @post_sq: flag for cqp db to ring
286 *
287 * This is called before connection establishment is started.
288 * For passive connections, when listener is created, it will
289 * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local
290 * ip address and tcp port. When SYN is received (passive
291 * connections) or sent (active connections), this routine is
292 * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED
293 * and quad is passed in info.
294 *
295 * When iwarp connection is done and its state moves to RTS, the
296 * quad hash entry in the hardware will point to iwarp's qp
297 * number and requires no calls from the driver.
298 */
299 static int
irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp * cqp,struct irdma_qhash_table_info * info,u64 scratch,bool post_sq)300 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,
301 struct irdma_qhash_table_info *info,
302 u64 scratch, bool post_sq)
303 {
304 __le64 *wqe;
305 u64 qw1 = 0;
306 u64 qw2 = 0;
307 u64 temp;
308 struct irdma_sc_vsi *vsi = info->vsi;
309
310 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
311 if (!wqe)
312 return -ENOMEM;
313
314 set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr));
315
316 qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |
317 FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);
318 if (info->ipv4_valid) {
319 set_64bit_val(wqe, 48,
320 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0]));
321 } else {
322 set_64bit_val(wqe, 56,
323 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) |
324 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1]));
325
326 set_64bit_val(wqe, 48,
327 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) |
328 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3]));
329 }
330 qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE,
331 vsi->qos[info->user_pri].qs_handle);
332 if (info->vlan_valid)
333 qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id);
334 set_64bit_val(wqe, 16, qw2);
335 if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
336 qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port);
337 if (!info->ipv4_valid) {
338 set_64bit_val(wqe, 40,
339 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) |
340 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1]));
341 set_64bit_val(wqe, 32,
342 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) |
343 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3]));
344 } else {
345 set_64bit_val(wqe, 32,
346 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0]));
347 }
348 }
349
350 set_64bit_val(wqe, 8, qw1);
351 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) |
352 FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE,
353 IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) |
354 FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) |
355 FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) |
356 FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) |
357 FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type);
358 dma_wmb(); /* make sure WQE is written before valid bit is set */
359
360 set_64bit_val(wqe, 24, temp);
361
362 print_hex_dump_debug("WQE: MANAGE_QHASH WQE", DUMP_PREFIX_OFFSET, 16,
363 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
364 if (post_sq)
365 irdma_sc_cqp_post_sq(cqp);
366
367 return 0;
368 }
369
370 /**
371 * irdma_sc_qp_init - initialize qp
372 * @qp: sc qp
373 * @info: initialization qp info
374 */
irdma_sc_qp_init(struct irdma_sc_qp * qp,struct irdma_qp_init_info * info)375 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info)
376 {
377 int ret_code;
378 u32 pble_obj_cnt;
379 u16 wqe_size;
380
381 if (info->qp_uk_init_info.max_sq_frag_cnt >
382 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||
383 info->qp_uk_init_info.max_rq_frag_cnt >
384 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)
385 return -EINVAL;
386
387 qp->dev = info->pd->dev;
388 qp->vsi = info->vsi;
389 qp->ieq_qp = info->vsi->exception_lan_q;
390 qp->sq_pa = info->sq_pa;
391 qp->rq_pa = info->rq_pa;
392 qp->hw_host_ctx_pa = info->host_ctx_pa;
393 qp->q2_pa = info->q2_pa;
394 qp->shadow_area_pa = info->shadow_area_pa;
395 qp->q2_buf = info->q2;
396 qp->pd = info->pd;
397 qp->hw_host_ctx = info->host_ctx;
398 info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
399 ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info);
400 if (ret_code)
401 return ret_code;
402
403 qp->virtual_map = info->virtual_map;
404 pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
405
406 if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
407 (info->virtual_map && info->rq_pa >= pble_obj_cnt))
408 return -EINVAL;
409
410 qp->llp_stream_handle = (void *)(-1);
411 qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
412 IRDMA_QUEUE_TYPE_SQ_RQ);
413 ibdev_dbg(to_ibdev(qp->dev),
414 "WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n",
415 qp->hw_sq_size, qp->qp_uk.sq_ring.size);
416 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)
417 wqe_size = IRDMA_WQE_SIZE_128;
418 else
419 ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
420 &wqe_size);
421 if (ret_code)
422 return ret_code;
423
424 qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *
425 (wqe_size / IRDMA_QP_WQE_MIN_SIZE), IRDMA_QUEUE_TYPE_SQ_RQ);
426 ibdev_dbg(to_ibdev(qp->dev),
427 "WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
428 qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
429 qp->sq_tph_val = info->sq_tph_val;
430 qp->rq_tph_val = info->rq_tph_val;
431 qp->sq_tph_en = info->sq_tph_en;
432 qp->rq_tph_en = info->rq_tph_en;
433 qp->rcv_tph_en = info->rcv_tph_en;
434 qp->xmit_tph_en = info->xmit_tph_en;
435 qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
436 qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
437
438 return 0;
439 }
440
441 /**
442 * irdma_sc_qp_create - create qp
443 * @qp: sc qp
444 * @info: qp create info
445 * @scratch: u64 saved to be used during cqp completion
446 * @post_sq: flag for cqp db to ring
447 */
irdma_sc_qp_create(struct irdma_sc_qp * qp,struct irdma_create_qp_info * info,u64 scratch,bool post_sq)448 int irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,
449 u64 scratch, bool post_sq)
450 {
451 struct irdma_sc_cqp *cqp;
452 __le64 *wqe;
453 u64 hdr;
454
455 cqp = qp->dev->cqp;
456 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
457 qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt)
458 return -EINVAL;
459
460 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
461 if (!wqe)
462 return -ENOMEM;
463
464 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
465 set_64bit_val(wqe, 40, qp->shadow_area_pa);
466
467 hdr = qp->qp_uk.qp_id |
468 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
469 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) |
470 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
471 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
472 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
473 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
474 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
475 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
476 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
477 info->arp_cache_idx_valid) |
478 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
480 dma_wmb(); /* make sure WQE is written before valid bit is set */
481
482 set_64bit_val(wqe, 24, hdr);
483
484 print_hex_dump_debug("WQE: QP_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
485 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
486 if (post_sq)
487 irdma_sc_cqp_post_sq(cqp);
488
489 return 0;
490 }
491
492 /**
493 * irdma_sc_qp_modify - modify qp cqp wqe
494 * @qp: sc qp
495 * @info: modify qp info
496 * @scratch: u64 saved to be used during cqp completion
497 * @post_sq: flag for cqp db to ring
498 */
irdma_sc_qp_modify(struct irdma_sc_qp * qp,struct irdma_modify_qp_info * info,u64 scratch,bool post_sq)499 int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info,
500 u64 scratch, bool post_sq)
501 {
502 __le64 *wqe;
503 struct irdma_sc_cqp *cqp;
504 u64 hdr;
505 u8 term_actions = 0;
506 u8 term_len = 0;
507
508 cqp = qp->dev->cqp;
509 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
510 if (!wqe)
511 return -ENOMEM;
512
513 if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {
514 if (info->dont_send_fin)
515 term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;
516 if (info->dont_send_term)
517 term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;
518 if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||
519 term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)
520 term_len = info->termlen;
521 }
522
523 set_64bit_val(wqe, 8,
524 FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) |
525 FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len));
526 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
527 set_64bit_val(wqe, 40, qp->shadow_area_pa);
528
529 hdr = qp->qp_uk.qp_id |
530 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) |
531 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |
532 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
533 FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID,
534 info->cached_var_valid) |
535 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
536 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
537 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
538 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
539 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
540 FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) |
541 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY,
542 info->remove_hash_idx) |
543 FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) |
544 FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) |
545 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
546 info->arp_cache_idx_valid) |
547 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
548 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
549 dma_wmb(); /* make sure WQE is written before valid bit is set */
550
551 set_64bit_val(wqe, 24, hdr);
552
553 print_hex_dump_debug("WQE: QP_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
554 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
555 if (post_sq)
556 irdma_sc_cqp_post_sq(cqp);
557
558 return 0;
559 }
560
561 /**
562 * irdma_sc_qp_destroy - cqp destroy qp
563 * @qp: sc qp
564 * @scratch: u64 saved to be used during cqp completion
565 * @remove_hash_idx: flag if to remove hash idx
566 * @ignore_mw_bnd: memory window bind flag
567 * @post_sq: flag for cqp db to ring
568 */
irdma_sc_qp_destroy(struct irdma_sc_qp * qp,u64 scratch,bool remove_hash_idx,bool ignore_mw_bnd,bool post_sq)569 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
570 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq)
571 {
572 __le64 *wqe;
573 struct irdma_sc_cqp *cqp;
574 u64 hdr;
575
576 cqp = qp->dev->cqp;
577 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
578 if (!wqe)
579 return -ENOMEM;
580
581 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
582 set_64bit_val(wqe, 40, qp->shadow_area_pa);
583
584 hdr = qp->qp_uk.qp_id |
585 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) |
586 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
587 FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) |
588 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) |
589 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
590 dma_wmb(); /* make sure WQE is written before valid bit is set */
591
592 set_64bit_val(wqe, 24, hdr);
593
594 print_hex_dump_debug("WQE: QP_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
595 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
596 if (post_sq)
597 irdma_sc_cqp_post_sq(cqp);
598
599 return 0;
600 }
601
602 /**
603 * irdma_sc_get_encoded_ird_size -
604 * @ird_size: IRD size
605 * The ird from the connection is rounded to a supported HW setting and then encoded
606 * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based
607 * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input
608 */
irdma_sc_get_encoded_ird_size(u16 ird_size)609 static u8 irdma_sc_get_encoded_ird_size(u16 ird_size)
610 {
611 switch (ird_size ?
612 roundup_pow_of_two(2 * ird_size) : 4) {
613 case 256:
614 return IRDMA_IRD_HW_SIZE_256;
615 case 128:
616 return IRDMA_IRD_HW_SIZE_128;
617 case 64:
618 case 32:
619 return IRDMA_IRD_HW_SIZE_64;
620 case 16:
621 case 8:
622 return IRDMA_IRD_HW_SIZE_16;
623 case 4:
624 default:
625 break;
626 }
627
628 return IRDMA_IRD_HW_SIZE_4;
629 }
630
631 /**
632 * irdma_sc_qp_setctx_roce - set qp's context
633 * @qp: sc qp
634 * @qp_ctx: context ptr
635 * @info: ctx info
636 */
irdma_sc_qp_setctx_roce(struct irdma_sc_qp * qp,__le64 * qp_ctx,struct irdma_qp_host_ctx_info * info)637 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
638 struct irdma_qp_host_ctx_info *info)
639 {
640 struct irdma_roce_offload_info *roce_info;
641 struct irdma_udp_offload_info *udp;
642 u8 push_mode_en;
643 u32 push_idx;
644
645 roce_info = info->roce_info;
646 udp = info->udp_info;
647 qp->user_pri = info->user_pri;
648 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
649 push_mode_en = 0;
650 push_idx = 0;
651 } else {
652 push_mode_en = 1;
653 push_idx = qp->push_idx;
654 }
655 set_64bit_val(qp_ctx, 0,
656 FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
657 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
658 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
659 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
660 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
661 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
662 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |
663 FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) |
664 FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) |
665 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) |
666 FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |
667 FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |
668 FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |
669 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag));
670 set_64bit_val(qp_ctx, 8, qp->sq_pa);
671 set_64bit_val(qp_ctx, 16, qp->rq_pa);
672 if ((roce_info->dcqcn_en || roce_info->dctcp_en) &&
673 !(udp->tos & 0x03))
674 udp->tos |= ECN_CODE_PT_VAL;
675 set_64bit_val(qp_ctx, 24,
676 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
677 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) |
678 FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) |
679 FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) |
680 FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port));
681 set_64bit_val(qp_ctx, 32,
682 FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) |
683 FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3]));
684 set_64bit_val(qp_ctx, 40,
685 FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) |
686 FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1]));
687 set_64bit_val(qp_ctx, 48,
688 FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) |
689 FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) |
690 FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx));
691 set_64bit_val(qp_ctx, 56,
692 FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) |
693 FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) |
694 FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) |
695 FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label));
696 set_64bit_val(qp_ctx, 64,
697 FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) |
698 FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp));
699 set_64bit_val(qp_ctx, 80,
700 FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) |
701 FIELD_PREP(IRDMAQPC_LSN, udp->lsn));
702 set_64bit_val(qp_ctx, 88,
703 FIELD_PREP(IRDMAQPC_EPSN, udp->epsn));
704 set_64bit_val(qp_ctx, 96,
705 FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) |
706 FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una));
707 set_64bit_val(qp_ctx, 112,
708 FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd));
709 set_64bit_val(qp_ctx, 128,
710 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) |
711 FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) |
712 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) |
713 FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin));
714 set_64bit_val(qp_ctx, 136,
715 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
716 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
717 set_64bit_val(qp_ctx, 144,
718 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
719 set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info->mac_addr) << 16);
720 set_64bit_val(qp_ctx, 160,
721 FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
722 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
723 FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
724 FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
725 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
726 FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
727 FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
728 FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
729 FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
730 FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) |
731 FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) |
732 FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |
733 FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));
734 set_64bit_val(qp_ctx, 168,
735 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
736 set_64bit_val(qp_ctx, 176,
737 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
738 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
739 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
740 set_64bit_val(qp_ctx, 184,
741 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) |
742 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2]));
743 set_64bit_val(qp_ctx, 192,
744 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) |
745 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));
746 set_64bit_val(qp_ctx, 200,
747 FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |
748 FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));
749 set_64bit_val(qp_ctx, 208,
750 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
751
752 print_hex_dump_debug("WQE: QP_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,
753 8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
754 }
755
756 /* irdma_sc_alloc_local_mac_entry - allocate a mac entry
757 * @cqp: struct for cqp hw
758 * @scratch: u64 saved to be used during cqp completion
759 * @post_sq: flag for cqp db to ring
760 */
irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp * cqp,u64 scratch,bool post_sq)761 static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
762 bool post_sq)
763 {
764 __le64 *wqe;
765 u64 hdr;
766
767 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
768 if (!wqe)
769 return -ENOMEM;
770
771 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
772 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) |
773 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
774
775 dma_wmb(); /* make sure WQE is written before valid bit is set */
776
777 set_64bit_val(wqe, 24, hdr);
778
779 print_hex_dump_debug("WQE: ALLOCATE_LOCAL_MAC WQE",
780 DUMP_PREFIX_OFFSET, 16, 8, wqe,
781 IRDMA_CQP_WQE_SIZE * 8, false);
782
783 if (post_sq)
784 irdma_sc_cqp_post_sq(cqp);
785 return 0;
786 }
787
788 /**
789 * irdma_sc_add_local_mac_entry - add mac enry
790 * @cqp: struct for cqp hw
791 * @info:mac addr info
792 * @scratch: u64 saved to be used during cqp completion
793 * @post_sq: flag for cqp db to ring
794 */
irdma_sc_add_local_mac_entry(struct irdma_sc_cqp * cqp,struct irdma_local_mac_entry_info * info,u64 scratch,bool post_sq)795 static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,
796 struct irdma_local_mac_entry_info *info,
797 u64 scratch, bool post_sq)
798 {
799 __le64 *wqe;
800 u64 header;
801
802 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
803 if (!wqe)
804 return -ENOMEM;
805
806 set_64bit_val(wqe, 32, ether_addr_to_u64(info->mac_addr));
807
808 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |
809 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
810 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
811 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
812
813 dma_wmb(); /* make sure WQE is written before valid bit is set */
814
815 set_64bit_val(wqe, 24, header);
816
817 print_hex_dump_debug("WQE: ADD_LOCAL_MAC WQE", DUMP_PREFIX_OFFSET, 16,
818 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
819
820 if (post_sq)
821 irdma_sc_cqp_post_sq(cqp);
822 return 0;
823 }
824
825 /**
826 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
827 * @cqp: struct for cqp hw
828 * @scratch: u64 saved to be used during cqp completion
829 * @entry_idx: index of mac entry
830 * @ignore_ref_count: to force mac adde delete
831 * @post_sq: flag for cqp db to ring
832 */
irdma_sc_del_local_mac_entry(struct irdma_sc_cqp * cqp,u64 scratch,u16 entry_idx,u8 ignore_ref_count,bool post_sq)833 static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
834 u16 entry_idx, u8 ignore_ref_count,
835 bool post_sq)
836 {
837 __le64 *wqe;
838 u64 header;
839
840 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
841 if (!wqe)
842 return -ENOMEM;
843 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) |
844 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
845 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
846 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) |
847 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
848 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count);
849
850 dma_wmb(); /* make sure WQE is written before valid bit is set */
851
852 set_64bit_val(wqe, 24, header);
853
854 print_hex_dump_debug("WQE: DEL_LOCAL_MAC_IPADDR WQE",
855 DUMP_PREFIX_OFFSET, 16, 8, wqe,
856 IRDMA_CQP_WQE_SIZE * 8, false);
857
858 if (post_sq)
859 irdma_sc_cqp_post_sq(cqp);
860 return 0;
861 }
862
863 /**
864 * irdma_sc_qp_setctx - set qp's context
865 * @qp: sc qp
866 * @qp_ctx: context ptr
867 * @info: ctx info
868 */
irdma_sc_qp_setctx(struct irdma_sc_qp * qp,__le64 * qp_ctx,struct irdma_qp_host_ctx_info * info)869 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
870 struct irdma_qp_host_ctx_info *info)
871 {
872 struct irdma_iwarp_offload_info *iw;
873 struct irdma_tcp_offload_info *tcp;
874 struct irdma_sc_dev *dev;
875 u8 push_mode_en;
876 u32 push_idx;
877 u64 qw0, qw3, qw7 = 0, qw16 = 0;
878 u64 mac = 0;
879
880 iw = info->iwarp_info;
881 tcp = info->tcp_info;
882 dev = qp->dev;
883 if (iw->rcv_mark_en) {
884 qp->pfpdu.marker_len = 4;
885 qp->pfpdu.rcv_start_seq = tcp->rcv_nxt;
886 }
887 qp->user_pri = info->user_pri;
888 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
889 push_mode_en = 0;
890 push_idx = 0;
891 } else {
892 push_mode_en = 1;
893 push_idx = qp->push_idx;
894 }
895 qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
896 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
897 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
898 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
899 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
900 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
901 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en);
902
903 set_64bit_val(qp_ctx, 8, qp->sq_pa);
904 set_64bit_val(qp_ctx, 16, qp->rq_pa);
905
906 qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
907 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size);
908 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
909 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX,
910 qp->src_mac_addr_idx);
911 set_64bit_val(qp_ctx, 136,
912 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
913 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
914 set_64bit_val(qp_ctx, 168,
915 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
916 set_64bit_val(qp_ctx, 176,
917 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
918 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
919 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) |
920 FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp));
921 if (info->iwarp_info_valid) {
922 qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) |
923 FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) |
924 FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) |
925 FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) |
926 FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) |
927 FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) |
928 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID,
929 iw->err_rq_idx_valid);
930 qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id);
931 qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) |
932 FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin);
933 set_64bit_val(qp_ctx, 144,
934 FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |
935 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
936
937 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
938 mac = ether_addr_to_u64(iw->mac_addr);
939
940 set_64bit_val(qp_ctx, 152,
941 mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));
942 set_64bit_val(qp_ctx, 160,
943 FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) |
944 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) |
945 FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
946 FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
947 FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
948 FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
949 FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
950 FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
951 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
952 FIELD_PREP(IRDMAQPC_IWARPMODE, 1) |
953 FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) |
954 FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) |
955 FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) |
956 FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nxt) |
957 FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nxt) |
958 FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en));
959 }
960 if (info->tcp_info_valid) {
961 qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) |
962 FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) |
963 FIELD_PREP(IRDMAQPC_INSERTVLANTAG,
964 tcp->insert_vlan_tag) |
965 FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) |
966 FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) |
967 FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) |
968 FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh);
969
970 if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))
971 tcp->tos |= ECN_CODE_PT_VAL;
972
973 qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) |
974 FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) |
975 FIELD_PREP(IRDMAQPC_TOS, tcp->tos) |
976 FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) |
977 FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port);
978 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
979 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx);
980
981 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
982 }
983 set_64bit_val(qp_ctx, 32,
984 FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) |
985 FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3]));
986 set_64bit_val(qp_ctx, 40,
987 FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) |
988 FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1]));
989 set_64bit_val(qp_ctx, 48,
990 FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) |
991 FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) |
992 FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) |
993 FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx));
994 qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) |
995 FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) |
996 FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT,
997 tcp->ignore_tcp_opt) |
998 FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT,
999 tcp->ignore_tcp_uns_opt) |
1000 FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) |
1001 FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) |
1002 FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale);
1003 set_64bit_val(qp_ctx, 72,
1004 FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) |
1005 FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age));
1006 set_64bit_val(qp_ctx, 80,
1007 FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) |
1008 FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd));
1009 set_64bit_val(qp_ctx, 88,
1010 FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) |
1011 FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd));
1012 set_64bit_val(qp_ctx, 96,
1013 FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) |
1014 FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una));
1015 set_64bit_val(qp_ctx, 104,
1016 FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) |
1017 FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var));
1018 set_64bit_val(qp_ctx, 112,
1019 FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) |
1020 FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd));
1021 set_64bit_val(qp_ctx, 120,
1022 FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) |
1023 FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2));
1024 qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) |
1025 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh);
1026 set_64bit_val(qp_ctx, 184,
1027 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) |
1028 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2]));
1029 set_64bit_val(qp_ctx, 192,
1030 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) |
1031 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0]));
1032 set_64bit_val(qp_ctx, 200,
1033 FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) |
1034 FIELD_PREP(IRDMAQPC_TLOW, iw->t_low));
1035 set_64bit_val(qp_ctx, 208,
1036 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
1037 }
1038
1039 set_64bit_val(qp_ctx, 0, qw0);
1040 set_64bit_val(qp_ctx, 24, qw3);
1041 set_64bit_val(qp_ctx, 56, qw7);
1042 set_64bit_val(qp_ctx, 128, qw16);
1043
1044 print_hex_dump_debug("WQE: QP_HOST CTX", DUMP_PREFIX_OFFSET, 16, 8,
1045 qp_ctx, IRDMA_QP_CTX_SIZE, false);
1046 }
1047
1048 /**
1049 * irdma_sc_alloc_stag - mr stag alloc
1050 * @dev: sc device struct
1051 * @info: stag info
1052 * @scratch: u64 saved to be used during cqp completion
1053 * @post_sq: flag for cqp db to ring
1054 */
irdma_sc_alloc_stag(struct irdma_sc_dev * dev,struct irdma_allocate_stag_info * info,u64 scratch,bool post_sq)1055 static int irdma_sc_alloc_stag(struct irdma_sc_dev *dev,
1056 struct irdma_allocate_stag_info *info,
1057 u64 scratch, bool post_sq)
1058 {
1059 __le64 *wqe;
1060 struct irdma_sc_cqp *cqp;
1061 u64 hdr;
1062 enum irdma_page_size page_size;
1063
1064 if (!info->total_len && !info->all_memory)
1065 return -EINVAL;
1066
1067 if (info->page_size == 0x40000000)
1068 page_size = IRDMA_PAGE_SIZE_1G;
1069 else if (info->page_size == 0x200000)
1070 page_size = IRDMA_PAGE_SIZE_2M;
1071 else
1072 page_size = IRDMA_PAGE_SIZE_4K;
1073
1074 cqp = dev->cqp;
1075 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1076 if (!wqe)
1077 return -ENOMEM;
1078
1079 set_64bit_val(wqe, 8,
1080 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
1081 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len));
1082 set_64bit_val(wqe, 16,
1083 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1084 set_64bit_val(wqe, 40,
1085 FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index));
1086
1087 if (info->chunk_size)
1088 set_64bit_val(wqe, 48,
1089 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx));
1090
1091 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1092 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1093 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1094 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1095 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1096 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |
1097 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1098 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1099 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1100 dma_wmb(); /* make sure WQE is written before valid bit is set */
1101
1102 set_64bit_val(wqe, 24, hdr);
1103
1104 print_hex_dump_debug("WQE: ALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1105 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1106 if (post_sq)
1107 irdma_sc_cqp_post_sq(cqp);
1108
1109 return 0;
1110 }
1111
1112 /**
1113 * irdma_sc_mr_reg_non_shared - non-shared mr registration
1114 * @dev: sc device struct
1115 * @info: mr info
1116 * @scratch: u64 saved to be used during cqp completion
1117 * @post_sq: flag for cqp db to ring
1118 */
irdma_sc_mr_reg_non_shared(struct irdma_sc_dev * dev,struct irdma_reg_ns_stag_info * info,u64 scratch,bool post_sq)1119 static int irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,
1120 struct irdma_reg_ns_stag_info *info,
1121 u64 scratch, bool post_sq)
1122 {
1123 __le64 *wqe;
1124 u64 fbo;
1125 struct irdma_sc_cqp *cqp;
1126 u64 hdr;
1127 u32 pble_obj_cnt;
1128 bool remote_access;
1129 u8 addr_type;
1130 enum irdma_page_size page_size;
1131
1132 if (!info->total_len && !info->all_memory)
1133 return -EINVAL;
1134
1135 if (info->page_size == 0x40000000)
1136 page_size = IRDMA_PAGE_SIZE_1G;
1137 else if (info->page_size == 0x200000)
1138 page_size = IRDMA_PAGE_SIZE_2M;
1139 else if (info->page_size == 0x1000)
1140 page_size = IRDMA_PAGE_SIZE_4K;
1141 else
1142 return -EINVAL;
1143
1144 if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
1145 IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
1146 remote_access = true;
1147 else
1148 remote_access = false;
1149
1150 pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
1151 if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)
1152 return -EINVAL;
1153
1154 cqp = dev->cqp;
1155 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1156 if (!wqe)
1157 return -ENOMEM;
1158 fbo = info->va & (info->page_size - 1);
1159
1160 set_64bit_val(wqe, 0,
1161 (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
1162 info->va : fbo));
1163 set_64bit_val(wqe, 8,
1164 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) |
1165 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1166 set_64bit_val(wqe, 16,
1167 FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) |
1168 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1169 if (!info->chunk_size) {
1170 set_64bit_val(wqe, 32, info->reg_addr_pa);
1171 set_64bit_val(wqe, 48, 0);
1172 } else {
1173 set_64bit_val(wqe, 32, 0);
1174 set_64bit_val(wqe, 48,
1175 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index));
1176 }
1177 set_64bit_val(wqe, 40, info->hmc_fcn_index);
1178 set_64bit_val(wqe, 56, 0);
1179
1180 addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
1181 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) |
1182 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1183 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1184 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1185 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1186 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) |
1187 FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |
1188 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1189 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1190 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1191 dma_wmb(); /* make sure WQE is written before valid bit is set */
1192
1193 set_64bit_val(wqe, 24, hdr);
1194
1195 print_hex_dump_debug("WQE: MR_REG_NS WQE", DUMP_PREFIX_OFFSET, 16, 8,
1196 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1197 if (post_sq)
1198 irdma_sc_cqp_post_sq(cqp);
1199
1200 return 0;
1201 }
1202
1203 /**
1204 * irdma_sc_dealloc_stag - deallocate stag
1205 * @dev: sc device struct
1206 * @info: dealloc stag info
1207 * @scratch: u64 saved to be used during cqp completion
1208 * @post_sq: flag for cqp db to ring
1209 */
irdma_sc_dealloc_stag(struct irdma_sc_dev * dev,struct irdma_dealloc_stag_info * info,u64 scratch,bool post_sq)1210 static int irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,
1211 struct irdma_dealloc_stag_info *info,
1212 u64 scratch, bool post_sq)
1213 {
1214 u64 hdr;
1215 __le64 *wqe;
1216 struct irdma_sc_cqp *cqp;
1217
1218 cqp = dev->cqp;
1219 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1220 if (!wqe)
1221 return -ENOMEM;
1222
1223 set_64bit_val(wqe, 8,
1224 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1225 set_64bit_val(wqe, 16,
1226 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1227
1228 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) |
1229 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) |
1230 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1231 dma_wmb(); /* make sure WQE is written before valid bit is set */
1232
1233 set_64bit_val(wqe, 24, hdr);
1234
1235 print_hex_dump_debug("WQE: DEALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16,
1236 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1237 if (post_sq)
1238 irdma_sc_cqp_post_sq(cqp);
1239
1240 return 0;
1241 }
1242
1243 /**
1244 * irdma_sc_mw_alloc - mw allocate
1245 * @dev: sc device struct
1246 * @info: memory window allocation information
1247 * @scratch: u64 saved to be used during cqp completion
1248 * @post_sq: flag for cqp db to ring
1249 */
irdma_sc_mw_alloc(struct irdma_sc_dev * dev,struct irdma_mw_alloc_info * info,u64 scratch,bool post_sq)1250 static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
1251 struct irdma_mw_alloc_info *info, u64 scratch,
1252 bool post_sq)
1253 {
1254 u64 hdr;
1255 struct irdma_sc_cqp *cqp;
1256 __le64 *wqe;
1257
1258 cqp = dev->cqp;
1259 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1260 if (!wqe)
1261 return -ENOMEM;
1262
1263 set_64bit_val(wqe, 8,
1264 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1265 set_64bit_val(wqe, 16,
1266 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index));
1267
1268 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1269 FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) |
1270 FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY,
1271 info->mw1_bind_dont_vldt_key) |
1272 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1273 dma_wmb(); /* make sure WQE is written before valid bit is set */
1274
1275 set_64bit_val(wqe, 24, hdr);
1276
1277 print_hex_dump_debug("WQE: MW_ALLOC WQE", DUMP_PREFIX_OFFSET, 16, 8,
1278 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1279 if (post_sq)
1280 irdma_sc_cqp_post_sq(cqp);
1281
1282 return 0;
1283 }
1284
1285 /**
1286 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
1287 * @qp: sc qp struct
1288 * @info: fast mr info
1289 * @post_sq: flag for cqp db to ring
1290 */
irdma_sc_mr_fast_register(struct irdma_sc_qp * qp,struct irdma_fast_reg_stag_info * info,bool post_sq)1291 int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
1292 struct irdma_fast_reg_stag_info *info,
1293 bool post_sq)
1294 {
1295 u64 temp, hdr;
1296 __le64 *wqe;
1297 u32 wqe_idx;
1298 enum irdma_page_size page_size;
1299 struct irdma_post_sq_info sq_info = {};
1300
1301 if (info->page_size == 0x40000000)
1302 page_size = IRDMA_PAGE_SIZE_1G;
1303 else if (info->page_size == 0x200000)
1304 page_size = IRDMA_PAGE_SIZE_2M;
1305 else
1306 page_size = IRDMA_PAGE_SIZE_4K;
1307
1308 sq_info.wr_id = info->wr_id;
1309 sq_info.signaled = info->signaled;
1310 sq_info.push_wqe = info->push_wqe;
1311
1312 wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
1313 IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
1314 if (!wqe)
1315 return -ENOMEM;
1316
1317 irdma_clr_wqes(&qp->qp_uk, wqe_idx);
1318
1319 ibdev_dbg(to_ibdev(qp->dev),
1320 "MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
1321 info->wr_id, wqe_idx,
1322 &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
1323
1324 temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
1325 (uintptr_t)info->va : info->fbo;
1326 set_64bit_val(wqe, 0, temp);
1327
1328 temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
1329 info->first_pm_pbl_index >> 16);
1330 set_64bit_val(wqe, 8,
1331 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |
1332 FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa));
1333 set_64bit_val(wqe, 16,
1334 info->total_len |
1335 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));
1336
1337 hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) |
1338 FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) |
1339 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) |
1340 FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
1341 FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
1342 FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
1343 FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
1344 FIELD_PREP(IRDMAQPSQ_PUSHWQE, (sq_info.push_wqe ? 1 : 0)) |
1345 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
1346 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
1347 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
1348 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1349 dma_wmb(); /* make sure WQE is written before valid bit is set */
1350
1351 set_64bit_val(wqe, 24, hdr);
1352
1353 print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1354 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1355 if (sq_info.push_wqe) {
1356 irdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,
1357 wqe_idx, post_sq);
1358 } else {
1359 if (post_sq)
1360 irdma_uk_qp_post_wr(&qp->qp_uk);
1361 }
1362
1363 return 0;
1364 }
1365
1366 /**
1367 * irdma_sc_gen_rts_ae - request AE generated after RTS
1368 * @qp: sc qp struct
1369 */
irdma_sc_gen_rts_ae(struct irdma_sc_qp * qp)1370 static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
1371 {
1372 __le64 *wqe;
1373 u64 hdr;
1374 struct irdma_qp_uk *qp_uk;
1375
1376 qp_uk = &qp->qp_uk;
1377
1378 wqe = qp_uk->sq_base[1].elem;
1379
1380 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
1381 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
1382 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1383 dma_wmb(); /* make sure WQE is written before valid bit is set */
1384
1385 set_64bit_val(wqe, 24, hdr);
1386 print_hex_dump_debug("QP: NOP W/LOCAL FENCE WQE", DUMP_PREFIX_OFFSET,
1387 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1388
1389 wqe = qp_uk->sq_base[2].elem;
1390 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) |
1391 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1392 dma_wmb(); /* make sure WQE is written before valid bit is set */
1393
1394 set_64bit_val(wqe, 24, hdr);
1395 print_hex_dump_debug("QP: CONN EST WQE", DUMP_PREFIX_OFFSET, 16, 8,
1396 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1397 }
1398
1399 /**
1400 * irdma_sc_send_lsmm - send last streaming mode message
1401 * @qp: sc qp struct
1402 * @lsmm_buf: buffer with lsmm message
1403 * @size: size of lsmm buffer
1404 * @stag: stag of lsmm buffer
1405 */
irdma_sc_send_lsmm(struct irdma_sc_qp * qp,void * lsmm_buf,u32 size,irdma_stag stag)1406 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1407 irdma_stag stag)
1408 {
1409 __le64 *wqe;
1410 u64 hdr;
1411 struct irdma_qp_uk *qp_uk;
1412
1413 qp_uk = &qp->qp_uk;
1414 wqe = qp_uk->sq_base->elem;
1415
1416 set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
1417 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1418 set_64bit_val(wqe, 8,
1419 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) |
1420 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag));
1421 } else {
1422 set_64bit_val(wqe, 8,
1423 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
1424 FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) |
1425 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1426 }
1427 set_64bit_val(wqe, 16, 0);
1428
1429 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
1430 FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
1431 FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
1432 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1433 dma_wmb(); /* make sure WQE is written before valid bit is set */
1434
1435 set_64bit_val(wqe, 24, hdr);
1436
1437 print_hex_dump_debug("WQE: SEND_LSMM WQE", DUMP_PREFIX_OFFSET, 16, 8,
1438 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1439
1440 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1441 irdma_sc_gen_rts_ae(qp);
1442 }
1443
1444 /**
1445 * irdma_sc_send_rtt - send last read0 or write0
1446 * @qp: sc qp struct
1447 * @read: Do read0 or write0
1448 */
irdma_sc_send_rtt(struct irdma_sc_qp * qp,bool read)1449 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
1450 {
1451 __le64 *wqe;
1452 u64 hdr;
1453 struct irdma_qp_uk *qp_uk;
1454
1455 qp_uk = &qp->qp_uk;
1456 wqe = qp_uk->sq_base->elem;
1457
1458 set_64bit_val(wqe, 0, 0);
1459 set_64bit_val(wqe, 16, 0);
1460 if (read) {
1461 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1462 set_64bit_val(wqe, 8,
1463 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd));
1464 } else {
1465 set_64bit_val(wqe, 8,
1466 (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1467 }
1468 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) |
1469 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) |
1470 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1471
1472 } else {
1473 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1474 set_64bit_val(wqe, 8, 0);
1475 } else {
1476 set_64bit_val(wqe, 8,
1477 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1478 }
1479 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) |
1480 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1481 }
1482
1483 dma_wmb(); /* make sure WQE is written before valid bit is set */
1484
1485 set_64bit_val(wqe, 24, hdr);
1486
1487 print_hex_dump_debug("WQE: RTR WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
1488 IRDMA_QP_WQE_MIN_SIZE, false);
1489
1490 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1491 irdma_sc_gen_rts_ae(qp);
1492 }
1493
1494 /**
1495 * irdma_iwarp_opcode - determine if incoming is rdma layer
1496 * @info: aeq info for the packet
1497 * @pkt: packet for error
1498 */
irdma_iwarp_opcode(struct irdma_aeqe_info * info,u8 * pkt)1499 static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)
1500 {
1501 __be16 *mpa;
1502 u32 opcode = 0xffffffff;
1503
1504 if (info->q2_data_written) {
1505 mpa = (__be16 *)pkt;
1506 opcode = ntohs(mpa[1]) & 0xf;
1507 }
1508
1509 return opcode;
1510 }
1511
1512 /**
1513 * irdma_locate_mpa - return pointer to mpa in the pkt
1514 * @pkt: packet with data
1515 */
irdma_locate_mpa(u8 * pkt)1516 static u8 *irdma_locate_mpa(u8 *pkt)
1517 {
1518 /* skip over ethernet header */
1519 pkt += IRDMA_MAC_HLEN;
1520
1521 /* Skip over IP and TCP headers */
1522 pkt += 4 * (pkt[0] & 0x0f);
1523 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
1524
1525 return pkt;
1526 }
1527
1528 /**
1529 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
1530 * @qp: sc qp ptr for pkt
1531 * @hdr: term hdr
1532 * @opcode: flush opcode for termhdr
1533 * @layer_etype: error layer + error type
1534 * @err: error cod ein the header
1535 */
irdma_bld_termhdr_ctrl(struct irdma_sc_qp * qp,struct irdma_terminate_hdr * hdr,enum irdma_flush_opcode opcode,u8 layer_etype,u8 err)1536 static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,
1537 struct irdma_terminate_hdr *hdr,
1538 enum irdma_flush_opcode opcode,
1539 u8 layer_etype, u8 err)
1540 {
1541 qp->flush_code = opcode;
1542 hdr->layer_etype = layer_etype;
1543 hdr->error_code = err;
1544 }
1545
1546 /**
1547 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
1548 * @pkt: ptr to mpa in offending pkt
1549 * @hdr: term hdr
1550 * @copy_len: offending pkt length to be copied to term hdr
1551 * @is_tagged: DDP tagged or untagged
1552 */
irdma_bld_termhdr_ddp_rdma(u8 * pkt,struct irdma_terminate_hdr * hdr,int * copy_len,u8 * is_tagged)1553 static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,
1554 int *copy_len, u8 *is_tagged)
1555 {
1556 u16 ddp_seg_len;
1557
1558 ddp_seg_len = ntohs(*(__be16 *)pkt);
1559 if (ddp_seg_len) {
1560 *copy_len = 2;
1561 hdr->hdrct = DDP_LEN_FLAG;
1562 if (pkt[2] & 0x80) {
1563 *is_tagged = 1;
1564 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
1565 *copy_len += TERM_DDP_LEN_TAGGED;
1566 hdr->hdrct |= DDP_HDR_FLAG;
1567 }
1568 } else {
1569 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
1570 *copy_len += TERM_DDP_LEN_UNTAGGED;
1571 hdr->hdrct |= DDP_HDR_FLAG;
1572 }
1573 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&
1574 ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {
1575 *copy_len += TERM_RDMA_LEN;
1576 hdr->hdrct |= RDMA_HDR_FLAG;
1577 }
1578 }
1579 }
1580 }
1581
1582 /**
1583 * irdma_bld_terminate_hdr - build terminate message header
1584 * @qp: qp associated with received terminate AE
1585 * @info: the struct contiaing AE information
1586 */
irdma_bld_terminate_hdr(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1587 static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
1588 struct irdma_aeqe_info *info)
1589 {
1590 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1591 int copy_len = 0;
1592 u8 is_tagged = 0;
1593 u32 opcode;
1594 struct irdma_terminate_hdr *termhdr;
1595
1596 termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;
1597 memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
1598
1599 if (info->q2_data_written) {
1600 pkt = irdma_locate_mpa(pkt);
1601 irdma_bld_termhdr_ddp_rdma(pkt, termhdr, ©_len, &is_tagged);
1602 }
1603
1604 opcode = irdma_iwarp_opcode(info, pkt);
1605 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1606 qp->sq_flush_code = info->sq;
1607 qp->rq_flush_code = info->rq;
1608
1609 switch (info->ae_id) {
1610 case IRDMA_AE_AMP_UNALLOCATED_STAG:
1611 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1612 if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)
1613 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1614 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1615 DDP_TAGGED_INV_STAG);
1616 else
1617 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1618 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1619 RDMAP_INV_STAG);
1620 break;
1621 case IRDMA_AE_AMP_BOUNDS_VIOLATION:
1622 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1623 if (info->q2_data_written)
1624 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1625 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1626 DDP_TAGGED_BOUNDS);
1627 else
1628 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1629 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1630 RDMAP_INV_BOUNDS);
1631 break;
1632 case IRDMA_AE_AMP_BAD_PD:
1633 switch (opcode) {
1634 case IRDMA_OP_TYPE_RDMA_WRITE:
1635 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1636 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1637 DDP_TAGGED_UNASSOC_STAG);
1638 break;
1639 case IRDMA_OP_TYPE_SEND_INV:
1640 case IRDMA_OP_TYPE_SEND_SOL_INV:
1641 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1642 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1643 RDMAP_CANT_INV_STAG);
1644 break;
1645 default:
1646 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1647 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1648 RDMAP_UNASSOC_STAG);
1649 }
1650 break;
1651 case IRDMA_AE_AMP_INVALID_STAG:
1652 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1653 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1654 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1655 RDMAP_INV_STAG);
1656 break;
1657 case IRDMA_AE_AMP_BAD_QP:
1658 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
1659 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1660 DDP_UNTAGGED_INV_QN);
1661 break;
1662 case IRDMA_AE_AMP_BAD_STAG_KEY:
1663 case IRDMA_AE_AMP_BAD_STAG_INDEX:
1664 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1665 switch (opcode) {
1666 case IRDMA_OP_TYPE_SEND_INV:
1667 case IRDMA_OP_TYPE_SEND_SOL_INV:
1668 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1669 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1670 RDMAP_CANT_INV_STAG);
1671 break;
1672 default:
1673 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1674 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1675 RDMAP_INV_STAG);
1676 }
1677 break;
1678 case IRDMA_AE_AMP_RIGHTS_VIOLATION:
1679 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
1680 case IRDMA_AE_PRIV_OPERATION_DENIED:
1681 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1682 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1683 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1684 RDMAP_ACCESS);
1685 break;
1686 case IRDMA_AE_AMP_TO_WRAP:
1687 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1688 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1689 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1690 RDMAP_TO_WRAP);
1691 break;
1692 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1693 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1694 (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
1695 break;
1696 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
1697 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1698 (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1699 DDP_CATASTROPHIC_LOCAL);
1700 break;
1701 case IRDMA_AE_LCE_QP_CATASTROPHIC:
1702 case IRDMA_AE_DDP_NO_L_BIT:
1703 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1704 (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1705 DDP_CATASTROPHIC_LOCAL);
1706 break;
1707 case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:
1708 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1709 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1710 DDP_UNTAGGED_INV_MSN_RANGE);
1711 break;
1712 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
1713 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1714 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1715 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1716 DDP_UNTAGGED_INV_TOO_LONG);
1717 break;
1718 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
1719 if (is_tagged)
1720 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1721 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1722 DDP_TAGGED_INV_DDP_VER);
1723 else
1724 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1725 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1726 DDP_UNTAGGED_INV_DDP_VER);
1727 break;
1728 case IRDMA_AE_DDP_UBE_INVALID_MO:
1729 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1730 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1731 DDP_UNTAGGED_INV_MO);
1732 break;
1733 case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
1734 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1735 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1736 DDP_UNTAGGED_INV_MSN_NO_BUF);
1737 break;
1738 case IRDMA_AE_DDP_UBE_INVALID_QN:
1739 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1740 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1741 DDP_UNTAGGED_INV_QN);
1742 break;
1743 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1744 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1745 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1746 RDMAP_INV_RDMAP_VER);
1747 break;
1748 default:
1749 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1750 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1751 RDMAP_UNSPECIFIED);
1752 break;
1753 }
1754
1755 if (copy_len)
1756 memcpy(termhdr + 1, pkt, copy_len);
1757
1758 return sizeof(struct irdma_terminate_hdr) + copy_len;
1759 }
1760
1761 /**
1762 * irdma_terminate_send_fin() - Send fin for terminate message
1763 * @qp: qp associated with received terminate AE
1764 */
irdma_terminate_send_fin(struct irdma_sc_qp * qp)1765 void irdma_terminate_send_fin(struct irdma_sc_qp *qp)
1766 {
1767 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1768 IRDMAQP_TERM_SEND_FIN_ONLY, 0);
1769 }
1770
1771 /**
1772 * irdma_terminate_connection() - Bad AE and send terminate to remote QP
1773 * @qp: qp associated with received terminate AE
1774 * @info: the struct contiaing AE information
1775 */
irdma_terminate_connection(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1776 void irdma_terminate_connection(struct irdma_sc_qp *qp,
1777 struct irdma_aeqe_info *info)
1778 {
1779 u8 termlen = 0;
1780
1781 if (qp->term_flags & IRDMA_TERM_SENT)
1782 return;
1783
1784 termlen = irdma_bld_terminate_hdr(qp, info);
1785 irdma_terminate_start_timer(qp);
1786 qp->term_flags |= IRDMA_TERM_SENT;
1787 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1788 IRDMAQP_TERM_SEND_TERM_ONLY, termlen);
1789 }
1790
1791 /**
1792 * irdma_terminate_received - handle terminate received AE
1793 * @qp: qp associated with received terminate AE
1794 * @info: the struct contiaing AE information
1795 */
irdma_terminate_received(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1796 void irdma_terminate_received(struct irdma_sc_qp *qp,
1797 struct irdma_aeqe_info *info)
1798 {
1799 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1800 __be32 *mpa;
1801 u8 ddp_ctl;
1802 u8 rdma_ctl;
1803 u16 aeq_id = 0;
1804 struct irdma_terminate_hdr *termhdr;
1805
1806 mpa = (__be32 *)irdma_locate_mpa(pkt);
1807 if (info->q2_data_written) {
1808 /* did not validate the frame - do it now */
1809 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
1810 rdma_ctl = ntohl(mpa[0]) & 0xff;
1811 if ((ddp_ctl & 0xc0) != 0x40)
1812 aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;
1813 else if ((ddp_ctl & 0x03) != 1)
1814 aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;
1815 else if (ntohl(mpa[2]) != 2)
1816 aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;
1817 else if (ntohl(mpa[3]) != 1)
1818 aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;
1819 else if (ntohl(mpa[4]) != 0)
1820 aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;
1821 else if ((rdma_ctl & 0xc0) != 0x40)
1822 aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
1823
1824 info->ae_id = aeq_id;
1825 if (info->ae_id) {
1826 /* Bad terminate recvd - send back a terminate */
1827 irdma_terminate_connection(qp, info);
1828 return;
1829 }
1830 }
1831
1832 qp->term_flags |= IRDMA_TERM_RCVD;
1833 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1834 termhdr = (struct irdma_terminate_hdr *)&mpa[5];
1835 if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
1836 termhdr->layer_etype == RDMAP_REMOTE_OP) {
1837 irdma_terminate_done(qp, 0);
1838 } else {
1839 irdma_terminate_start_timer(qp);
1840 irdma_terminate_send_fin(qp);
1841 }
1842 }
1843
irdma_null_ws_add(struct irdma_sc_vsi * vsi,u8 user_pri)1844 static int irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
1845 {
1846 return 0;
1847 }
1848
irdma_null_ws_remove(struct irdma_sc_vsi * vsi,u8 user_pri)1849 static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
1850 {
1851 /* do nothing */
1852 }
1853
irdma_null_ws_reset(struct irdma_sc_vsi * vsi)1854 static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)
1855 {
1856 /* do nothing */
1857 }
1858
1859 /**
1860 * irdma_sc_vsi_init - Init the vsi structure
1861 * @vsi: pointer to vsi structure to initialize
1862 * @info: the info used to initialize the vsi struct
1863 */
irdma_sc_vsi_init(struct irdma_sc_vsi * vsi,struct irdma_vsi_init_info * info)1864 void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
1865 struct irdma_vsi_init_info *info)
1866 {
1867 int i;
1868
1869 vsi->dev = info->dev;
1870 vsi->back_vsi = info->back_vsi;
1871 vsi->register_qset = info->register_qset;
1872 vsi->unregister_qset = info->unregister_qset;
1873 vsi->mtu = info->params->mtu;
1874 vsi->exception_lan_q = info->exception_lan_q;
1875 vsi->vsi_idx = info->pf_data_vsi_num;
1876 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1877 vsi->fcn_id = info->dev->hmc_fn_id;
1878
1879 irdma_set_qos_info(vsi, info->params);
1880 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
1881 mutex_init(&vsi->qos[i].qos_mutex);
1882 INIT_LIST_HEAD(&vsi->qos[i].qplist);
1883 }
1884 if (vsi->register_qset) {
1885 vsi->dev->ws_add = irdma_ws_add;
1886 vsi->dev->ws_remove = irdma_ws_remove;
1887 vsi->dev->ws_reset = irdma_ws_reset;
1888 } else {
1889 vsi->dev->ws_add = irdma_null_ws_add;
1890 vsi->dev->ws_remove = irdma_null_ws_remove;
1891 vsi->dev->ws_reset = irdma_null_ws_reset;
1892 }
1893 }
1894
1895 /**
1896 * irdma_get_fcn_id - Return the function id
1897 * @vsi: pointer to the vsi
1898 */
irdma_get_fcn_id(struct irdma_sc_vsi * vsi)1899 static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
1900 {
1901 struct irdma_stats_inst_info stats_info = {};
1902 struct irdma_sc_dev *dev = vsi->dev;
1903 u8 fcn_id = IRDMA_INVALID_FCN_ID;
1904 u8 start_idx, max_stats, i;
1905
1906 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
1907 if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
1908 &stats_info))
1909 return stats_info.stats_idx;
1910 }
1911
1912 start_idx = 1;
1913 max_stats = 16;
1914 for (i = start_idx; i < max_stats; i++)
1915 if (!dev->fcn_id_array[i]) {
1916 fcn_id = i;
1917 dev->fcn_id_array[i] = true;
1918 break;
1919 }
1920
1921 return fcn_id;
1922 }
1923
1924 /**
1925 * irdma_vsi_stats_init - Initialize the vsi statistics
1926 * @vsi: pointer to the vsi structure
1927 * @info: The info structure used for initialization
1928 */
irdma_vsi_stats_init(struct irdma_sc_vsi * vsi,struct irdma_vsi_stats_info * info)1929 int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
1930 struct irdma_vsi_stats_info *info)
1931 {
1932 u8 fcn_id = info->fcn_id;
1933 struct irdma_dma_mem *stats_buff_mem;
1934
1935 vsi->pestat = info->pestat;
1936 vsi->pestat->hw = vsi->dev->hw;
1937 vsi->pestat->vsi = vsi;
1938 stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;
1939 stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);
1940 stats_buff_mem->va = dma_alloc_coherent(vsi->pestat->hw->device,
1941 stats_buff_mem->size,
1942 &stats_buff_mem->pa,
1943 GFP_KERNEL);
1944 if (!stats_buff_mem->va)
1945 return -ENOMEM;
1946
1947 vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va;
1948 vsi->pestat->gather_info.last_gather_stats_va =
1949 (void *)((uintptr_t)stats_buff_mem->va +
1950 IRDMA_GATHER_STATS_BUF_SIZE);
1951
1952 irdma_hw_stats_start_timer(vsi);
1953 if (info->alloc_fcn_id)
1954 fcn_id = irdma_get_fcn_id(vsi);
1955 if (fcn_id == IRDMA_INVALID_FCN_ID)
1956 goto stats_error;
1957
1958 vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
1959 vsi->fcn_id = fcn_id;
1960 if (info->alloc_fcn_id) {
1961 vsi->pestat->gather_info.use_stats_inst = true;
1962 vsi->pestat->gather_info.stats_inst_index = fcn_id;
1963 }
1964
1965 return 0;
1966
1967 stats_error:
1968 dma_free_coherent(vsi->pestat->hw->device, stats_buff_mem->size,
1969 stats_buff_mem->va, stats_buff_mem->pa);
1970 stats_buff_mem->va = NULL;
1971
1972 return -EIO;
1973 }
1974
1975 /**
1976 * irdma_vsi_stats_free - Free the vsi stats
1977 * @vsi: pointer to the vsi structure
1978 */
irdma_vsi_stats_free(struct irdma_sc_vsi * vsi)1979 void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
1980 {
1981 struct irdma_stats_inst_info stats_info = {};
1982 u8 fcn_id = vsi->fcn_id;
1983 struct irdma_sc_dev *dev = vsi->dev;
1984
1985 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
1986 if (vsi->stats_fcn_id_alloc) {
1987 stats_info.stats_idx = vsi->fcn_id;
1988 irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
1989 &stats_info);
1990 }
1991 } else {
1992 if (vsi->stats_fcn_id_alloc &&
1993 fcn_id < vsi->dev->hw_attrs.max_stat_inst)
1994 vsi->dev->fcn_id_array[fcn_id] = false;
1995 }
1996
1997 if (!vsi->pestat)
1998 return;
1999 irdma_hw_stats_stop_timer(vsi);
2000 dma_free_coherent(vsi->pestat->hw->device,
2001 vsi->pestat->gather_info.stats_buff_mem.size,
2002 vsi->pestat->gather_info.stats_buff_mem.va,
2003 vsi->pestat->gather_info.stats_buff_mem.pa);
2004 vsi->pestat->gather_info.stats_buff_mem.va = NULL;
2005 }
2006
2007 /**
2008 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
2009 * @wqsize: size of the wq (sq, rq) to encoded_size
2010 * @queue_type: queue type selected for the calculation algorithm
2011 */
irdma_get_encoded_wqe_size(u32 wqsize,enum irdma_queue_type queue_type)2012 u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type)
2013 {
2014 u8 encoded_size = 0;
2015
2016 /* cqp sq's hw coded value starts from 1 for size of 4
2017 * while it starts from 0 for qp' wq's.
2018 */
2019 if (queue_type == IRDMA_QUEUE_TYPE_CQP)
2020 encoded_size = 1;
2021 wqsize >>= 2;
2022 while (wqsize >>= 1)
2023 encoded_size++;
2024
2025 return encoded_size;
2026 }
2027
2028 /**
2029 * irdma_sc_gather_stats - collect the statistics
2030 * @cqp: struct for cqp hw
2031 * @info: gather stats info structure
2032 * @scratch: u64 saved to be used during cqp completion
2033 */
irdma_sc_gather_stats(struct irdma_sc_cqp * cqp,struct irdma_stats_gather_info * info,u64 scratch)2034 static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
2035 struct irdma_stats_gather_info *info,
2036 u64 scratch)
2037 {
2038 __le64 *wqe;
2039 u64 temp;
2040
2041 if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)
2042 return -ENOMEM;
2043
2044 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2045 if (!wqe)
2046 return -ENOMEM;
2047
2048 set_64bit_val(wqe, 40,
2049 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index));
2050 set_64bit_val(wqe, 32, info->stats_buff_mem.pa);
2051
2052 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2053 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) |
2054 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX,
2055 info->stats_inst_index) |
2056 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2057 info->use_hmc_fcn_index) |
2058 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS);
2059 dma_wmb(); /* make sure WQE is written before valid bit is set */
2060
2061 set_64bit_val(wqe, 24, temp);
2062
2063 print_hex_dump_debug("STATS: GATHER_STATS WQE", DUMP_PREFIX_OFFSET,
2064 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2065
2066 irdma_sc_cqp_post_sq(cqp);
2067 ibdev_dbg(to_ibdev(cqp->dev),
2068 "STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
2069 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
2070
2071 return 0;
2072 }
2073
2074 /**
2075 * irdma_sc_manage_stats_inst - allocate or free stats instance
2076 * @cqp: struct for cqp hw
2077 * @info: stats info structure
2078 * @alloc: alloc vs. delete flag
2079 * @scratch: u64 saved to be used during cqp completion
2080 */
irdma_sc_manage_stats_inst(struct irdma_sc_cqp * cqp,struct irdma_stats_inst_info * info,bool alloc,u64 scratch)2081 static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,
2082 struct irdma_stats_inst_info *info,
2083 bool alloc, u64 scratch)
2084 {
2085 __le64 *wqe;
2086 u64 temp;
2087
2088 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2089 if (!wqe)
2090 return -ENOMEM;
2091
2092 set_64bit_val(wqe, 40,
2093 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id));
2094 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2095 FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) |
2096 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2097 info->use_hmc_fcn_index) |
2098 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) |
2099 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS);
2100
2101 dma_wmb(); /* make sure WQE is written before valid bit is set */
2102
2103 set_64bit_val(wqe, 24, temp);
2104
2105 print_hex_dump_debug("WQE: MANAGE_STATS WQE", DUMP_PREFIX_OFFSET, 16,
2106 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2107
2108 irdma_sc_cqp_post_sq(cqp);
2109 return 0;
2110 }
2111
2112 /**
2113 * irdma_sc_set_up_map - set the up map table
2114 * @cqp: struct for cqp hw
2115 * @info: User priority map info
2116 * @scratch: u64 saved to be used during cqp completion
2117 */
irdma_sc_set_up_map(struct irdma_sc_cqp * cqp,struct irdma_up_info * info,u64 scratch)2118 static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,
2119 struct irdma_up_info *info, u64 scratch)
2120 {
2121 __le64 *wqe;
2122 u64 temp = 0;
2123 int i;
2124
2125 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2126 if (!wqe)
2127 return -ENOMEM;
2128
2129 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
2130 temp |= (u64)info->map[i] << (i * 8);
2131
2132 set_64bit_val(wqe, 0, temp);
2133 set_64bit_val(wqe, 40,
2134 FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) |
2135 FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx));
2136
2137 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) |
2138 FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) |
2139 FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE,
2140 info->use_cnp_up_override) |
2141 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP);
2142 dma_wmb(); /* make sure WQE is written before valid bit is set */
2143
2144 set_64bit_val(wqe, 24, temp);
2145
2146 print_hex_dump_debug("WQE: UPMAP WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
2147 IRDMA_CQP_WQE_SIZE * 8, false);
2148 irdma_sc_cqp_post_sq(cqp);
2149
2150 return 0;
2151 }
2152
2153 /**
2154 * irdma_sc_manage_ws_node - create/modify/destroy WS node
2155 * @cqp: struct for cqp hw
2156 * @info: node info structure
2157 * @node_op: 0 for add 1 for modify, 2 for delete
2158 * @scratch: u64 saved to be used during cqp completion
2159 */
irdma_sc_manage_ws_node(struct irdma_sc_cqp * cqp,struct irdma_ws_node_info * info,enum irdma_ws_node_op node_op,u64 scratch)2160 static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
2161 struct irdma_ws_node_info *info,
2162 enum irdma_ws_node_op node_op, u64 scratch)
2163 {
2164 __le64 *wqe;
2165 u64 temp = 0;
2166
2167 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2168 if (!wqe)
2169 return -ENOMEM;
2170
2171 set_64bit_val(wqe, 32,
2172 FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) |
2173 FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight));
2174
2175 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) |
2176 FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) |
2177 FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) |
2178 FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) |
2179 FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) |
2180 FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) |
2181 FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) |
2182 FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) |
2183 FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id);
2184 dma_wmb(); /* make sure WQE is written before valid bit is set */
2185
2186 set_64bit_val(wqe, 24, temp);
2187
2188 print_hex_dump_debug("WQE: MANAGE_WS WQE", DUMP_PREFIX_OFFSET, 16, 8,
2189 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2190 irdma_sc_cqp_post_sq(cqp);
2191
2192 return 0;
2193 }
2194
2195 /**
2196 * irdma_sc_qp_flush_wqes - flush qp's wqe
2197 * @qp: sc qp
2198 * @info: dlush information
2199 * @scratch: u64 saved to be used during cqp completion
2200 * @post_sq: flag for cqp db to ring
2201 */
irdma_sc_qp_flush_wqes(struct irdma_sc_qp * qp,struct irdma_qp_flush_info * info,u64 scratch,bool post_sq)2202 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
2203 struct irdma_qp_flush_info *info, u64 scratch,
2204 bool post_sq)
2205 {
2206 u64 temp = 0;
2207 __le64 *wqe;
2208 struct irdma_sc_cqp *cqp;
2209 u64 hdr;
2210 bool flush_sq = false, flush_rq = false;
2211
2212 if (info->rq && !qp->flush_rq)
2213 flush_rq = true;
2214 if (info->sq && !qp->flush_sq)
2215 flush_sq = true;
2216 qp->flush_sq |= flush_sq;
2217 qp->flush_rq |= flush_rq;
2218
2219 if (!flush_sq && !flush_rq) {
2220 ibdev_dbg(to_ibdev(qp->dev),
2221 "CQP: Additional flush request ignored for qp %x\n",
2222 qp->qp_uk.qp_id);
2223 return -EALREADY;
2224 }
2225
2226 cqp = qp->pd->dev->cqp;
2227 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2228 if (!wqe)
2229 return -ENOMEM;
2230
2231 if (info->userflushcode) {
2232 if (flush_rq)
2233 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR,
2234 info->rq_minor_code) |
2235 FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR,
2236 info->rq_major_code);
2237 if (flush_sq)
2238 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR,
2239 info->sq_minor_code) |
2240 FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR,
2241 info->sq_major_code);
2242 }
2243 set_64bit_val(wqe, 16, temp);
2244
2245 temp = (info->generate_ae) ?
2246 info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2247 info->ae_src) : 0;
2248 set_64bit_val(wqe, 8, temp);
2249
2250 hdr = qp->qp_uk.qp_id |
2251 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) |
2252 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) |
2253 FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) |
2254 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) |
2255 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) |
2256 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2257 dma_wmb(); /* make sure WQE is written before valid bit is set */
2258
2259 set_64bit_val(wqe, 24, hdr);
2260
2261 print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
2262 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2263 if (post_sq)
2264 irdma_sc_cqp_post_sq(cqp);
2265
2266 return 0;
2267 }
2268
2269 /**
2270 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
2271 * @qp: sc qp
2272 * @info: gen ae information
2273 * @scratch: u64 saved to be used during cqp completion
2274 * @post_sq: flag for cqp db to ring
2275 */
irdma_sc_gen_ae(struct irdma_sc_qp * qp,struct irdma_gen_ae_info * info,u64 scratch,bool post_sq)2276 static int irdma_sc_gen_ae(struct irdma_sc_qp *qp,
2277 struct irdma_gen_ae_info *info, u64 scratch,
2278 bool post_sq)
2279 {
2280 u64 temp;
2281 __le64 *wqe;
2282 struct irdma_sc_cqp *cqp;
2283 u64 hdr;
2284
2285 cqp = qp->pd->dev->cqp;
2286 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2287 if (!wqe)
2288 return -ENOMEM;
2289
2290 temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2291 info->ae_src);
2292 set_64bit_val(wqe, 8, temp);
2293
2294 hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,
2295 IRDMA_CQP_OP_GEN_AE) |
2296 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) |
2297 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2298 dma_wmb(); /* make sure WQE is written before valid bit is set */
2299
2300 set_64bit_val(wqe, 24, hdr);
2301
2302 print_hex_dump_debug("WQE: GEN_AE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2303 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2304 if (post_sq)
2305 irdma_sc_cqp_post_sq(cqp);
2306
2307 return 0;
2308 }
2309
2310 /*** irdma_sc_qp_upload_context - upload qp's context
2311 * @dev: sc device struct
2312 * @info: upload context info ptr for return
2313 * @scratch: u64 saved to be used during cqp completion
2314 * @post_sq: flag for cqp db to ring
2315 */
irdma_sc_qp_upload_context(struct irdma_sc_dev * dev,struct irdma_upload_context_info * info,u64 scratch,bool post_sq)2316 static int irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,
2317 struct irdma_upload_context_info *info,
2318 u64 scratch, bool post_sq)
2319 {
2320 __le64 *wqe;
2321 struct irdma_sc_cqp *cqp;
2322 u64 hdr;
2323
2324 cqp = dev->cqp;
2325 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2326 if (!wqe)
2327 return -ENOMEM;
2328
2329 set_64bit_val(wqe, 16, info->buf_pa);
2330
2331 hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) |
2332 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) |
2333 FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) |
2334 FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) |
2335 FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) |
2336 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2337 dma_wmb(); /* make sure WQE is written before valid bit is set */
2338
2339 set_64bit_val(wqe, 24, hdr);
2340
2341 print_hex_dump_debug("WQE: QP_UPLOAD_CTX WQE", DUMP_PREFIX_OFFSET, 16,
2342 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2343 if (post_sq)
2344 irdma_sc_cqp_post_sq(cqp);
2345
2346 return 0;
2347 }
2348
2349 /**
2350 * irdma_sc_manage_push_page - Handle push page
2351 * @cqp: struct for cqp hw
2352 * @info: push page info
2353 * @scratch: u64 saved to be used during cqp completion
2354 * @post_sq: flag for cqp db to ring
2355 */
irdma_sc_manage_push_page(struct irdma_sc_cqp * cqp,struct irdma_cqp_manage_push_page_info * info,u64 scratch,bool post_sq)2356 static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,
2357 struct irdma_cqp_manage_push_page_info *info,
2358 u64 scratch, bool post_sq)
2359 {
2360 __le64 *wqe;
2361 u64 hdr;
2362
2363 if (info->free_page &&
2364 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)
2365 return -EINVAL;
2366
2367 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2368 if (!wqe)
2369 return -ENOMEM;
2370
2371 set_64bit_val(wqe, 16, info->qs_handle);
2372 hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) |
2373 FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) |
2374 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) |
2375 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
2376 FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page);
2377 dma_wmb(); /* make sure WQE is written before valid bit is set */
2378
2379 set_64bit_val(wqe, 24, hdr);
2380
2381 print_hex_dump_debug("WQE: MANAGE_PUSH_PAGES WQE", DUMP_PREFIX_OFFSET,
2382 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2383 if (post_sq)
2384 irdma_sc_cqp_post_sq(cqp);
2385
2386 return 0;
2387 }
2388
2389 /**
2390 * irdma_sc_suspend_qp - suspend qp for param change
2391 * @cqp: struct for cqp hw
2392 * @qp: sc qp struct
2393 * @scratch: u64 saved to be used during cqp completion
2394 */
irdma_sc_suspend_qp(struct irdma_sc_cqp * cqp,struct irdma_sc_qp * qp,u64 scratch)2395 static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,
2396 u64 scratch)
2397 {
2398 u64 hdr;
2399 __le64 *wqe;
2400
2401 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2402 if (!wqe)
2403 return -ENOMEM;
2404
2405 hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) |
2406 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) |
2407 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2408 dma_wmb(); /* make sure WQE is written before valid bit is set */
2409
2410 set_64bit_val(wqe, 24, hdr);
2411
2412 print_hex_dump_debug("WQE: SUSPEND_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2413 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2414 irdma_sc_cqp_post_sq(cqp);
2415
2416 return 0;
2417 }
2418
2419 /**
2420 * irdma_sc_resume_qp - resume qp after suspend
2421 * @cqp: struct for cqp hw
2422 * @qp: sc qp struct
2423 * @scratch: u64 saved to be used during cqp completion
2424 */
irdma_sc_resume_qp(struct irdma_sc_cqp * cqp,struct irdma_sc_qp * qp,u64 scratch)2425 static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,
2426 u64 scratch)
2427 {
2428 u64 hdr;
2429 __le64 *wqe;
2430
2431 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2432 if (!wqe)
2433 return -ENOMEM;
2434
2435 set_64bit_val(wqe, 16,
2436 FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle));
2437
2438 hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) |
2439 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) |
2440 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2441 dma_wmb(); /* make sure WQE is written before valid bit is set */
2442
2443 set_64bit_val(wqe, 24, hdr);
2444
2445 print_hex_dump_debug("WQE: RESUME_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2446 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2447 irdma_sc_cqp_post_sq(cqp);
2448
2449 return 0;
2450 }
2451
2452 /**
2453 * irdma_sc_cq_ack - acknowledge completion q
2454 * @cq: cq struct
2455 */
irdma_sc_cq_ack(struct irdma_sc_cq * cq)2456 static inline void irdma_sc_cq_ack(struct irdma_sc_cq *cq)
2457 {
2458 writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);
2459 }
2460
2461 /**
2462 * irdma_sc_cq_init - initialize completion q
2463 * @cq: cq struct
2464 * @info: cq initialization info
2465 */
irdma_sc_cq_init(struct irdma_sc_cq * cq,struct irdma_cq_init_info * info)2466 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info)
2467 {
2468 u32 pble_obj_cnt;
2469
2470 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2471 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
2472 return -EINVAL;
2473
2474 cq->cq_pa = info->cq_base_pa;
2475 cq->dev = info->dev;
2476 cq->ceq_id = info->ceq_id;
2477 info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;
2478 info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;
2479 irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info);
2480
2481 cq->virtual_map = info->virtual_map;
2482 cq->pbl_chunk_size = info->pbl_chunk_size;
2483 cq->ceqe_mask = info->ceqe_mask;
2484 cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;
2485 cq->shadow_area_pa = info->shadow_area_pa;
2486 cq->shadow_read_threshold = info->shadow_read_threshold;
2487 cq->ceq_id_valid = info->ceq_id_valid;
2488 cq->tph_en = info->tph_en;
2489 cq->tph_val = info->tph_val;
2490 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2491 cq->vsi = info->vsi;
2492
2493 return 0;
2494 }
2495
2496 /**
2497 * irdma_sc_cq_create - create completion q
2498 * @cq: cq struct
2499 * @scratch: u64 saved to be used during cqp completion
2500 * @check_overflow: flag for overflow check
2501 * @post_sq: flag for cqp db to ring
2502 */
irdma_sc_cq_create(struct irdma_sc_cq * cq,u64 scratch,bool check_overflow,bool post_sq)2503 static int irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch,
2504 bool check_overflow, bool post_sq)
2505 {
2506 __le64 *wqe;
2507 struct irdma_sc_cqp *cqp;
2508 u64 hdr;
2509 struct irdma_sc_ceq *ceq;
2510 int ret_code = 0;
2511
2512 cqp = cq->dev->cqp;
2513 if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt)
2514 return -EINVAL;
2515
2516 if (cq->ceq_id >= cq->dev->hmc_fpm_misc.max_ceqs)
2517 return -EINVAL;
2518
2519 ceq = cq->dev->ceq[cq->ceq_id];
2520 if (ceq && ceq->reg_cq)
2521 ret_code = irdma_sc_add_cq_ctx(ceq, cq);
2522
2523 if (ret_code)
2524 return ret_code;
2525
2526 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2527 if (!wqe) {
2528 if (ceq && ceq->reg_cq)
2529 irdma_sc_remove_cq_ctx(ceq, cq);
2530 return -ENOMEM;
2531 }
2532
2533 set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2534 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2535 set_64bit_val(wqe, 16,
2536 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
2537 set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2538 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2539 set_64bit_val(wqe, 48,
2540 FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
2541 set_64bit_val(wqe, 56,
2542 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2543 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2544
2545 hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
2546 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2547 IRDMA_CQPSQ_CQ_CEQID) |
2548 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
2549 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2550 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) |
2551 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2552 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2553 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2554 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2555 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2556 cq->cq_uk.avoid_mem_cflct) |
2557 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2558 dma_wmb(); /* make sure WQE is written before valid bit is set */
2559
2560 set_64bit_val(wqe, 24, hdr);
2561
2562 print_hex_dump_debug("WQE: CQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2563 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2564 if (post_sq)
2565 irdma_sc_cqp_post_sq(cqp);
2566
2567 return 0;
2568 }
2569
2570 /**
2571 * irdma_sc_cq_destroy - destroy completion q
2572 * @cq: cq struct
2573 * @scratch: u64 saved to be used during cqp completion
2574 * @post_sq: flag for cqp db to ring
2575 */
irdma_sc_cq_destroy(struct irdma_sc_cq * cq,u64 scratch,bool post_sq)2576 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq)
2577 {
2578 struct irdma_sc_cqp *cqp;
2579 __le64 *wqe;
2580 u64 hdr;
2581 struct irdma_sc_ceq *ceq;
2582
2583 cqp = cq->dev->cqp;
2584 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2585 if (!wqe)
2586 return -ENOMEM;
2587
2588 ceq = cq->dev->ceq[cq->ceq_id];
2589 if (ceq && ceq->reg_cq)
2590 irdma_sc_remove_cq_ctx(ceq, cq);
2591
2592 set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2593 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2594 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2595 set_64bit_val(wqe, 48,
2596 (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2597
2598 hdr = cq->cq_uk.cq_id |
2599 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2600 IRDMA_CQPSQ_CQ_CEQID) |
2601 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
2602 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2603 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2604 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2605 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2606 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2607 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) |
2608 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2609 dma_wmb(); /* make sure WQE is written before valid bit is set */
2610
2611 set_64bit_val(wqe, 24, hdr);
2612
2613 print_hex_dump_debug("WQE: CQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2614 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2615 if (post_sq)
2616 irdma_sc_cqp_post_sq(cqp);
2617
2618 return 0;
2619 }
2620
2621 /**
2622 * irdma_sc_cq_resize - set resized cq buffer info
2623 * @cq: resized cq
2624 * @info: resized cq buffer info
2625 */
irdma_sc_cq_resize(struct irdma_sc_cq * cq,struct irdma_modify_cq_info * info)2626 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)
2627 {
2628 cq->virtual_map = info->virtual_map;
2629 cq->cq_pa = info->cq_pa;
2630 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2631 cq->pbl_chunk_size = info->pbl_chunk_size;
2632 irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);
2633 }
2634
2635 /**
2636 * irdma_sc_cq_modify - modify a Completion Queue
2637 * @cq: cq struct
2638 * @info: modification info struct
2639 * @scratch: u64 saved to be used during cqp completion
2640 * @post_sq: flag to post to sq
2641 */
irdma_sc_cq_modify(struct irdma_sc_cq * cq,struct irdma_modify_cq_info * info,u64 scratch,bool post_sq)2642 static int irdma_sc_cq_modify(struct irdma_sc_cq *cq,
2643 struct irdma_modify_cq_info *info, u64 scratch,
2644 bool post_sq)
2645 {
2646 struct irdma_sc_cqp *cqp;
2647 __le64 *wqe;
2648 u64 hdr;
2649 u32 pble_obj_cnt;
2650
2651 pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2652 if (info->cq_resize && info->virtual_map &&
2653 info->first_pm_pbl_idx >= pble_obj_cnt)
2654 return -EINVAL;
2655
2656 cqp = cq->dev->cqp;
2657 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2658 if (!wqe)
2659 return -ENOMEM;
2660
2661 set_64bit_val(wqe, 0, info->cq_size);
2662 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2663 set_64bit_val(wqe, 16,
2664 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold));
2665 set_64bit_val(wqe, 32, info->cq_pa);
2666 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2667 set_64bit_val(wqe, 48, info->first_pm_pbl_idx);
2668 set_64bit_val(wqe, 56,
2669 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2670 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2671
2672 hdr = cq->cq_uk.cq_id |
2673 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) |
2674 FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) |
2675 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) |
2676 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) |
2677 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) |
2678 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2679 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2680 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2681 cq->cq_uk.avoid_mem_cflct) |
2682 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2683 dma_wmb(); /* make sure WQE is written before valid bit is set */
2684
2685 set_64bit_val(wqe, 24, hdr);
2686
2687 print_hex_dump_debug("WQE: CQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2688 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2689 if (post_sq)
2690 irdma_sc_cqp_post_sq(cqp);
2691
2692 return 0;
2693 }
2694
2695 /**
2696 * irdma_check_cqp_progress - check cqp processing progress
2697 * @timeout: timeout info struct
2698 * @dev: sc device struct
2699 */
irdma_check_cqp_progress(struct irdma_cqp_timeout * timeout,struct irdma_sc_dev * dev)2700 void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)
2701 {
2702 u64 completed_ops = atomic64_read(&dev->cqp->completed_ops);
2703
2704 if (timeout->compl_cqp_cmds != completed_ops) {
2705 timeout->compl_cqp_cmds = completed_ops;
2706 timeout->count = 0;
2707 } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) {
2708 timeout->count++;
2709 }
2710 }
2711
2712 /**
2713 * irdma_get_cqp_reg_info - get head and tail for cqp using registers
2714 * @cqp: struct for cqp hw
2715 * @val: cqp tail register value
2716 * @tail: wqtail register value
2717 * @error: cqp processing err
2718 */
irdma_get_cqp_reg_info(struct irdma_sc_cqp * cqp,u32 * val,u32 * tail,u32 * error)2719 static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,
2720 u32 *tail, u32 *error)
2721 {
2722 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]);
2723 *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val);
2724 *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val);
2725 }
2726
2727 /**
2728 * irdma_cqp_poll_registers - poll cqp registers
2729 * @cqp: struct for cqp hw
2730 * @tail: wqtail register value
2731 * @count: how many times to try for completion
2732 */
irdma_cqp_poll_registers(struct irdma_sc_cqp * cqp,u32 tail,u32 count)2733 static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail,
2734 u32 count)
2735 {
2736 u32 i = 0;
2737 u32 newtail, error, val;
2738
2739 while (i++ < count) {
2740 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error);
2741 if (error) {
2742 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
2743 ibdev_dbg(to_ibdev(cqp->dev),
2744 "CQP: CQPERRCODES error_code[x%08X]\n",
2745 error);
2746 return -EIO;
2747 }
2748 if (newtail != tail) {
2749 /* SUCCESS */
2750 IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
2751 atomic64_inc(&cqp->completed_ops);
2752 return 0;
2753 }
2754 udelay(cqp->dev->hw_attrs.max_sleep_count);
2755 }
2756
2757 return -ETIMEDOUT;
2758 }
2759
2760 /**
2761 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
2762 * @dev: sc device struct
2763 * @buf: pointer to commit buffer
2764 * @buf_idx: buffer index
2765 * @obj_info: object info pointer
2766 * @rsrc_idx: indexs of memory resource
2767 */
irdma_sc_decode_fpm_commit(struct irdma_sc_dev * dev,__le64 * buf,u32 buf_idx,struct irdma_hmc_obj_info * obj_info,u32 rsrc_idx)2768 static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 *buf,
2769 u32 buf_idx, struct irdma_hmc_obj_info *obj_info,
2770 u32 rsrc_idx)
2771 {
2772 u64 temp;
2773
2774 get_64bit_val(buf, buf_idx, &temp);
2775
2776 switch (rsrc_idx) {
2777 case IRDMA_HMC_IW_QP:
2778 obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp);
2779 break;
2780 case IRDMA_HMC_IW_CQ:
2781 obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT);
2782 break;
2783 case IRDMA_HMC_IW_APBVT_ENTRY:
2784 obj_info[rsrc_idx].cnt = 1;
2785 break;
2786 default:
2787 obj_info[rsrc_idx].cnt = (u32)temp;
2788 break;
2789 }
2790
2791 obj_info[rsrc_idx].base = (temp >> IRDMA_COMMIT_FPM_BASE_S) * 512;
2792
2793 return temp;
2794 }
2795
2796 /**
2797 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
2798 * @dev: pointer to dev struct
2799 * @buf: ptr to fpm commit buffer
2800 * @info: ptr to irdma_hmc_obj_info struct
2801 * @sd: number of SDs for HMC objects
2802 *
2803 * parses fpm commit info and copy base value
2804 * of hmc objects in hmc_info
2805 */
2806 static void
irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev * dev,__le64 * buf,struct irdma_hmc_obj_info * info,u32 * sd)2807 irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf,
2808 struct irdma_hmc_obj_info *info, u32 *sd)
2809 {
2810 u64 size;
2811 u32 i;
2812 u64 max_base = 0;
2813 u32 last_hmc_obj = 0;
2814
2815 irdma_sc_decode_fpm_commit(dev, buf, 0, info,
2816 IRDMA_HMC_IW_QP);
2817 irdma_sc_decode_fpm_commit(dev, buf, 8, info,
2818 IRDMA_HMC_IW_CQ);
2819 /* skiping RSRVD */
2820 irdma_sc_decode_fpm_commit(dev, buf, 24, info,
2821 IRDMA_HMC_IW_HTE);
2822 irdma_sc_decode_fpm_commit(dev, buf, 32, info,
2823 IRDMA_HMC_IW_ARP);
2824 irdma_sc_decode_fpm_commit(dev, buf, 40, info,
2825 IRDMA_HMC_IW_APBVT_ENTRY);
2826 irdma_sc_decode_fpm_commit(dev, buf, 48, info,
2827 IRDMA_HMC_IW_MR);
2828 irdma_sc_decode_fpm_commit(dev, buf, 56, info,
2829 IRDMA_HMC_IW_XF);
2830 irdma_sc_decode_fpm_commit(dev, buf, 64, info,
2831 IRDMA_HMC_IW_XFFL);
2832 irdma_sc_decode_fpm_commit(dev, buf, 72, info,
2833 IRDMA_HMC_IW_Q1);
2834 irdma_sc_decode_fpm_commit(dev, buf, 80, info,
2835 IRDMA_HMC_IW_Q1FL);
2836 irdma_sc_decode_fpm_commit(dev, buf, 88, info,
2837 IRDMA_HMC_IW_TIMER);
2838 irdma_sc_decode_fpm_commit(dev, buf, 112, info,
2839 IRDMA_HMC_IW_PBLE);
2840 /* skipping RSVD. */
2841 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
2842 irdma_sc_decode_fpm_commit(dev, buf, 96, info,
2843 IRDMA_HMC_IW_FSIMC);
2844 irdma_sc_decode_fpm_commit(dev, buf, 104, info,
2845 IRDMA_HMC_IW_FSIAV);
2846 irdma_sc_decode_fpm_commit(dev, buf, 128, info,
2847 IRDMA_HMC_IW_RRF);
2848 irdma_sc_decode_fpm_commit(dev, buf, 136, info,
2849 IRDMA_HMC_IW_RRFFL);
2850 irdma_sc_decode_fpm_commit(dev, buf, 144, info,
2851 IRDMA_HMC_IW_HDR);
2852 irdma_sc_decode_fpm_commit(dev, buf, 152, info,
2853 IRDMA_HMC_IW_MD);
2854 irdma_sc_decode_fpm_commit(dev, buf, 160, info,
2855 IRDMA_HMC_IW_OOISC);
2856 irdma_sc_decode_fpm_commit(dev, buf, 168, info,
2857 IRDMA_HMC_IW_OOISCFFL);
2858 }
2859
2860 /* searching for the last object in HMC to find the size of the HMC area. */
2861 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
2862 if (info[i].base > max_base) {
2863 max_base = info[i].base;
2864 last_hmc_obj = i;
2865 }
2866 }
2867
2868 size = info[last_hmc_obj].cnt * info[last_hmc_obj].size +
2869 info[last_hmc_obj].base;
2870
2871 if (size & 0x1FFFFF)
2872 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
2873 else
2874 *sd = (u32)(size >> 21);
2875
2876 }
2877
2878 /**
2879 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
2880 * @buf: ptr to fpm query buffer
2881 * @buf_idx: index into buf
2882 * @obj_info: ptr to irdma_hmc_obj_info struct
2883 * @rsrc_idx: resource index into info
2884 *
2885 * Decode a 64 bit value from fpm query buffer into max count and size
2886 */
irdma_sc_decode_fpm_query(__le64 * buf,u32 buf_idx,struct irdma_hmc_obj_info * obj_info,u32 rsrc_idx)2887 static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx,
2888 struct irdma_hmc_obj_info *obj_info,
2889 u32 rsrc_idx)
2890 {
2891 u64 temp;
2892 u32 size;
2893
2894 get_64bit_val(buf, buf_idx, &temp);
2895 obj_info[rsrc_idx].max_cnt = (u32)temp;
2896 size = (u32)(temp >> 32);
2897 obj_info[rsrc_idx].size = BIT_ULL(size);
2898
2899 return temp;
2900 }
2901
2902 /**
2903 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
2904 * @dev: ptr to shared code device
2905 * @buf: ptr to fpm query buffer
2906 * @hmc_info: ptr to irdma_hmc_obj_info struct
2907 * @hmc_fpm_misc: ptr to fpm data
2908 *
2909 * parses fpm query buffer and copy max_cnt and
2910 * size value of hmc objects in hmc_info
2911 */
irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev * dev,__le64 * buf,struct irdma_hmc_info * hmc_info,struct irdma_hmc_fpm_misc * hmc_fpm_misc)2912 static int irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,
2913 struct irdma_hmc_info *hmc_info,
2914 struct irdma_hmc_fpm_misc *hmc_fpm_misc)
2915 {
2916 struct irdma_hmc_obj_info *obj_info;
2917 u64 temp;
2918 u32 size;
2919 u16 max_pe_sds;
2920
2921 obj_info = hmc_info->hmc_obj;
2922
2923 get_64bit_val(buf, 0, &temp);
2924 hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);
2925 max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp);
2926
2927 hmc_fpm_misc->max_sds = max_pe_sds;
2928 hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
2929 get_64bit_val(buf, 8, &temp);
2930 obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp);
2931 size = (u32)(temp >> 32);
2932 obj_info[IRDMA_HMC_IW_QP].size = BIT_ULL(size);
2933
2934 get_64bit_val(buf, 16, &temp);
2935 obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp);
2936 size = (u32)(temp >> 32);
2937 obj_info[IRDMA_HMC_IW_CQ].size = BIT_ULL(size);
2938
2939 irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);
2940 irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);
2941
2942 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;
2943 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;
2944
2945 irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);
2946 irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);
2947
2948 get_64bit_val(buf, 64, &temp);
2949 obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;
2950 obj_info[IRDMA_HMC_IW_XFFL].size = 4;
2951 hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);
2952 if (!hmc_fpm_misc->xf_block_size)
2953 return -EINVAL;
2954
2955 irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);
2956 get_64bit_val(buf, 80, &temp);
2957 obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;
2958 obj_info[IRDMA_HMC_IW_Q1FL].size = 4;
2959
2960 hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);
2961 if (!hmc_fpm_misc->q1_block_size)
2962 return -EINVAL;
2963
2964 irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);
2965
2966 get_64bit_val(buf, 112, &temp);
2967 obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;
2968 obj_info[IRDMA_HMC_IW_PBLE].size = 8;
2969
2970 get_64bit_val(buf, 120, &temp);
2971 hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);
2972 hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);
2973 hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);
2974 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
2975 return 0;
2976 irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);
2977 irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);
2978 irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);
2979
2980 get_64bit_val(buf, 136, &temp);
2981 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;
2982 obj_info[IRDMA_HMC_IW_RRFFL].size = 4;
2983 hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);
2984 if (!hmc_fpm_misc->rrf_block_size &&
2985 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)
2986 return -EINVAL;
2987
2988 irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
2989 irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
2990 irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
2991
2992 get_64bit_val(buf, 168, &temp);
2993 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
2994 obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
2995 hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
2996 if (!hmc_fpm_misc->ooiscf_block_size &&
2997 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
2998 return -EINVAL;
2999
3000 return 0;
3001 }
3002
3003 /**
3004 * irdma_sc_find_reg_cq - find cq ctx index
3005 * @ceq: ceq sc structure
3006 * @cq: cq sc structure
3007 */
irdma_sc_find_reg_cq(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3008 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,
3009 struct irdma_sc_cq *cq)
3010 {
3011 u32 i;
3012
3013 for (i = 0; i < ceq->reg_cq_size; i++) {
3014 if (cq == ceq->reg_cq[i])
3015 return i;
3016 }
3017
3018 return IRDMA_INVALID_CQ_IDX;
3019 }
3020
3021 /**
3022 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
3023 * @ceq: ceq sc structure
3024 * @cq: cq sc structure
3025 */
irdma_sc_add_cq_ctx(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3026 int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
3027 {
3028 unsigned long flags;
3029
3030 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3031
3032 if (ceq->reg_cq_size == ceq->elem_cnt) {
3033 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3034 return -ENOMEM;
3035 }
3036
3037 ceq->reg_cq[ceq->reg_cq_size++] = cq;
3038
3039 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3040
3041 return 0;
3042 }
3043
3044 /**
3045 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
3046 * @ceq: ceq sc structure
3047 * @cq: cq sc structure
3048 */
irdma_sc_remove_cq_ctx(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3049 void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
3050 {
3051 unsigned long flags;
3052 u32 cq_ctx_idx;
3053
3054 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3055 cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);
3056 if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX)
3057 goto exit;
3058
3059 ceq->reg_cq_size--;
3060 if (cq_ctx_idx != ceq->reg_cq_size)
3061 ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];
3062 ceq->reg_cq[ceq->reg_cq_size] = NULL;
3063
3064 exit:
3065 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3066 }
3067
3068 /**
3069 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
3070 * @cqp: IWARP control queue pair pointer
3071 * @info: IWARP control queue pair init info pointer
3072 *
3073 * Initializes the object and context buffers for a control Queue Pair.
3074 */
irdma_sc_cqp_init(struct irdma_sc_cqp * cqp,struct irdma_cqp_init_info * info)3075 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
3076 struct irdma_cqp_init_info *info)
3077 {
3078 u8 hw_sq_size;
3079
3080 if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||
3081 info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||
3082 ((info->sq_size & (info->sq_size - 1))))
3083 return -EINVAL;
3084
3085 hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size,
3086 IRDMA_QUEUE_TYPE_CQP);
3087 cqp->size = sizeof(*cqp);
3088 cqp->sq_size = info->sq_size;
3089 cqp->hw_sq_size = hw_sq_size;
3090 cqp->sq_base = info->sq;
3091 cqp->host_ctx = info->host_ctx;
3092 cqp->sq_pa = info->sq_pa;
3093 cqp->host_ctx_pa = info->host_ctx_pa;
3094 cqp->dev = info->dev;
3095 cqp->struct_ver = info->struct_ver;
3096 cqp->hw_maj_ver = info->hw_maj_ver;
3097 cqp->hw_min_ver = info->hw_min_ver;
3098 cqp->scratch_array = info->scratch_array;
3099 cqp->polarity = 0;
3100 cqp->en_datacenter_tcp = info->en_datacenter_tcp;
3101 cqp->ena_vf_count = info->ena_vf_count;
3102 cqp->hmc_profile = info->hmc_profile;
3103 cqp->ceqs_per_vf = info->ceqs_per_vf;
3104 cqp->disable_packed = info->disable_packed;
3105 cqp->rocev2_rto_policy = info->rocev2_rto_policy;
3106 cqp->protocol_used = info->protocol_used;
3107 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params));
3108 info->dev->cqp = cqp;
3109
3110 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
3111 cqp->requested_ops = 0;
3112 atomic64_set(&cqp->completed_ops, 0);
3113 /* for the cqp commands backlog. */
3114 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
3115
3116 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]);
3117 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]);
3118 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3119
3120 ibdev_dbg(to_ibdev(cqp->dev),
3121 "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n",
3122 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base,
3123 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity);
3124 return 0;
3125 }
3126
3127 /**
3128 * irdma_sc_cqp_create - create cqp during bringup
3129 * @cqp: struct for cqp hw
3130 * @maj_err: If error, major err number
3131 * @min_err: If error, minor err number
3132 */
irdma_sc_cqp_create(struct irdma_sc_cqp * cqp,u16 * maj_err,u16 * min_err)3133 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err)
3134 {
3135 u64 temp;
3136 u8 hw_rev;
3137 u32 cnt = 0, p1, p2, val = 0, err_code;
3138 int ret_code;
3139
3140 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev;
3141 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,
3142 IRDMA_SD_BUF_ALIGNMENT);
3143 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device,
3144 cqp->sdbuf.size, &cqp->sdbuf.pa,
3145 GFP_KERNEL);
3146 if (!cqp->sdbuf.va)
3147 return -ENOMEM;
3148
3149 spin_lock_init(&cqp->dev->cqp_lock);
3150
3151 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) |
3152 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) |
3153 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) |
3154 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf);
3155 if (hw_rev >= IRDMA_GEN_2) {
3156 temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY,
3157 cqp->rocev2_rto_policy) |
3158 FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED,
3159 cqp->protocol_used);
3160 }
3161
3162 set_64bit_val(cqp->host_ctx, 0, temp);
3163 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
3164
3165 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
3166 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
3167 set_64bit_val(cqp->host_ctx, 16, temp);
3168 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
3169 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) |
3170 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver);
3171 if (hw_rev >= IRDMA_GEN_2) {
3172 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) |
3173 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor);
3174 }
3175 set_64bit_val(cqp->host_ctx, 32, temp);
3176 set_64bit_val(cqp->host_ctx, 40, 0);
3177 temp = 0;
3178 if (hw_rev >= IRDMA_GEN_2) {
3179 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) |
3180 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) |
3181 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor);
3182 }
3183 set_64bit_val(cqp->host_ctx, 48, temp);
3184 temp = 0;
3185 if (hw_rev >= IRDMA_GEN_2) {
3186 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) |
3187 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) |
3188 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) |
3189 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod);
3190 }
3191 set_64bit_val(cqp->host_ctx, 56, temp);
3192 print_hex_dump_debug("WQE: CQP_HOST_CTX WQE", DUMP_PREFIX_OFFSET, 16,
3193 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false);
3194 p1 = cqp->host_ctx_pa >> 32;
3195 p2 = (u32)cqp->host_ctx_pa;
3196
3197 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3198 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3199
3200 do {
3201 if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3202 ret_code = -ETIMEDOUT;
3203 goto err;
3204 }
3205 udelay(cqp->dev->hw_attrs.max_sleep_count);
3206 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3207 } while (!val);
3208
3209 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {
3210 ret_code = -EOPNOTSUPP;
3211 goto err;
3212 }
3213
3214 cqp->process_cqp_sds = irdma_update_sds_noccq;
3215 return 0;
3216
3217 err:
3218 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3219 cqp->sdbuf.va, cqp->sdbuf.pa);
3220 cqp->sdbuf.va = NULL;
3221 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3222 *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code);
3223 *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code);
3224 return ret_code;
3225 }
3226
3227 /**
3228 * irdma_sc_cqp_post_sq - post of cqp's sq
3229 * @cqp: struct for cqp hw
3230 */
irdma_sc_cqp_post_sq(struct irdma_sc_cqp * cqp)3231 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)
3232 {
3233 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);
3234
3235 ibdev_dbg(to_ibdev(cqp->dev),
3236 "WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
3237 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
3238 }
3239
3240 /**
3241 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
3242 * and pass back index
3243 * @cqp: CQP HW structure
3244 * @scratch: private data for CQP WQE
3245 * @wqe_idx: WQE index of CQP SQ
3246 */
irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp * cqp,u64 scratch,u32 * wqe_idx)3247 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
3248 u32 *wqe_idx)
3249 {
3250 __le64 *wqe = NULL;
3251 int ret_code;
3252
3253 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {
3254 ibdev_dbg(to_ibdev(cqp->dev),
3255 "WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n",
3256 cqp->sq_ring.head, cqp->sq_ring.tail,
3257 cqp->sq_ring.size);
3258 return NULL;
3259 }
3260 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
3261 if (ret_code)
3262 return NULL;
3263
3264 cqp->requested_ops++;
3265 if (!*wqe_idx)
3266 cqp->polarity = !cqp->polarity;
3267 wqe = cqp->sq_base[*wqe_idx].elem;
3268 cqp->scratch_array[*wqe_idx] = scratch;
3269 IRDMA_CQP_INIT_WQE(wqe);
3270
3271 return wqe;
3272 }
3273
3274 /**
3275 * irdma_sc_cqp_destroy - destroy cqp during close
3276 * @cqp: struct for cqp hw
3277 */
irdma_sc_cqp_destroy(struct irdma_sc_cqp * cqp)3278 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)
3279 {
3280 u32 cnt = 0, val;
3281 int ret_code = 0;
3282
3283 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3284 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3285 do {
3286 if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3287 ret_code = -ETIMEDOUT;
3288 break;
3289 }
3290 udelay(cqp->dev->hw_attrs.max_sleep_count);
3291 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3292 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
3293
3294 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3295 cqp->sdbuf.va, cqp->sdbuf.pa);
3296 cqp->sdbuf.va = NULL;
3297 return ret_code;
3298 }
3299
3300 /**
3301 * irdma_sc_ccq_arm - enable intr for control cq
3302 * @ccq: ccq sc struct
3303 */
irdma_sc_ccq_arm(struct irdma_sc_cq * ccq)3304 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
3305 {
3306 u64 temp_val;
3307 u16 sw_cq_sel;
3308 u8 arm_next_se;
3309 u8 arm_seq_num;
3310
3311 get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
3312 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
3313 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
3314 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
3315 arm_seq_num++;
3316 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
3317 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
3318 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
3319 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
3320 set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
3321
3322 dma_wmb(); /* make sure shadow area is updated before arming */
3323
3324 writel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db);
3325 }
3326
3327 /**
3328 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
3329 * @ccq: ccq sc struct
3330 * @info: completion q entry to return
3331 */
irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq * ccq,struct irdma_ccq_cqe_info * info)3332 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
3333 struct irdma_ccq_cqe_info *info)
3334 {
3335 u64 qp_ctx, temp, temp1;
3336 __le64 *cqe;
3337 struct irdma_sc_cqp *cqp;
3338 u32 wqe_idx;
3339 u32 error;
3340 u8 polarity;
3341 int ret_code = 0;
3342
3343 if (ccq->cq_uk.avoid_mem_cflct)
3344 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
3345 else
3346 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);
3347
3348 get_64bit_val(cqe, 24, &temp);
3349 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp);
3350 if (polarity != ccq->cq_uk.polarity)
3351 return -ENOENT;
3352
3353 /* Ensure CEQE contents are read after valid bit is checked */
3354 dma_rmb();
3355
3356 get_64bit_val(cqe, 8, &qp_ctx);
3357 cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;
3358 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
3359 info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;
3360 info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);
3361 if (info->error) {
3362 info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);
3363 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3364 ibdev_dbg(to_ibdev(cqp->dev),
3365 "CQP: CQPERRCODES error_code[x%08X]\n", error);
3366 }
3367
3368 wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp);
3369 info->scratch = cqp->scratch_array[wqe_idx];
3370
3371 get_64bit_val(cqe, 16, &temp1);
3372 info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);
3373 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
3374 info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);
3375 info->cqp = cqp;
3376
3377 /* move the head for cq */
3378 IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
3379 if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring))
3380 ccq->cq_uk.polarity ^= 1;
3381
3382 /* update cq tail in cq shadow memory also */
3383 IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
3384 set_64bit_val(ccq->cq_uk.shadow_area, 0,
3385 IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));
3386
3387 dma_wmb(); /* make sure shadow area is updated before moving tail */
3388
3389 IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
3390 atomic64_inc(&cqp->completed_ops);
3391
3392 return ret_code;
3393 }
3394
3395 /**
3396 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
3397 * @cqp: struct for cqp hw
3398 * @op_code: cqp opcode for completion
3399 * @compl_info: completion q entry to return
3400 */
irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp * cqp,u8 op_code,struct irdma_ccq_cqe_info * compl_info)3401 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code,
3402 struct irdma_ccq_cqe_info *compl_info)
3403 {
3404 struct irdma_ccq_cqe_info info = {};
3405 struct irdma_sc_cq *ccq;
3406 int ret_code = 0;
3407 u32 cnt = 0;
3408
3409 ccq = cqp->dev->ccq;
3410 while (1) {
3411 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)
3412 return -ETIMEDOUT;
3413
3414 if (irdma_sc_ccq_get_cqe_info(ccq, &info)) {
3415 udelay(cqp->dev->hw_attrs.max_sleep_count);
3416 continue;
3417 }
3418 if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) {
3419 ret_code = -EIO;
3420 break;
3421 }
3422 /* make sure op code matches*/
3423 if (op_code == info.op_code)
3424 break;
3425 ibdev_dbg(to_ibdev(cqp->dev),
3426 "WQE: opcode mismatch for my op code 0x%x, returned opcode %x\n",
3427 op_code, info.op_code);
3428 }
3429
3430 if (compl_info)
3431 memcpy(compl_info, &info, sizeof(*compl_info));
3432
3433 return ret_code;
3434 }
3435
3436 /**
3437 * irdma_sc_manage_hmc_pm_func_table - manage of function table
3438 * @cqp: struct for cqp hw
3439 * @scratch: u64 saved to be used during cqp completion
3440 * @info: info for the manage function table operation
3441 * @post_sq: flag for cqp db to ring
3442 */
irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp * cqp,struct irdma_hmc_fcn_info * info,u64 scratch,bool post_sq)3443 static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp,
3444 struct irdma_hmc_fcn_info *info,
3445 u64 scratch, bool post_sq)
3446 {
3447 __le64 *wqe;
3448 u64 hdr;
3449
3450 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3451 if (!wqe)
3452 return -ENOMEM;
3453
3454 set_64bit_val(wqe, 0, 0);
3455 set_64bit_val(wqe, 8, 0);
3456 set_64bit_val(wqe, 16, 0);
3457 set_64bit_val(wqe, 32, 0);
3458 set_64bit_val(wqe, 40, 0);
3459 set_64bit_val(wqe, 48, 0);
3460 set_64bit_val(wqe, 56, 0);
3461
3462 hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) |
3463 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
3464 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) |
3465 FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) |
3466 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3467 dma_wmb(); /* make sure WQE is written before valid bit is set */
3468
3469 set_64bit_val(wqe, 24, hdr);
3470
3471 print_hex_dump_debug("WQE: MANAGE_HMC_PM_FUNC_TABLE WQE",
3472 DUMP_PREFIX_OFFSET, 16, 8, wqe,
3473 IRDMA_CQP_WQE_SIZE * 8, false);
3474 if (post_sq)
3475 irdma_sc_cqp_post_sq(cqp);
3476
3477 return 0;
3478 }
3479
3480 /**
3481 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion
3482 * for fpm commit
3483 * @cqp: struct for cqp hw
3484 */
irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp * cqp)3485 static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)
3486 {
3487 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL,
3488 NULL);
3489 }
3490
3491 /**
3492 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values
3493 * @cqp: struct for cqp hw
3494 * @scratch: u64 saved to be used during cqp completion
3495 * @hmc_fn_id: hmc function id
3496 * @commit_fpm_mem: Memory for fpm values
3497 * @post_sq: flag for cqp db to ring
3498 * @wait_type: poll ccq or cqp registers for cqp completion
3499 */
irdma_sc_commit_fpm_val(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,struct irdma_dma_mem * commit_fpm_mem,bool post_sq,u8 wait_type)3500 static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
3501 u8 hmc_fn_id,
3502 struct irdma_dma_mem *commit_fpm_mem,
3503 bool post_sq, u8 wait_type)
3504 {
3505 __le64 *wqe;
3506 u64 hdr;
3507 u32 tail, val, error;
3508 int ret_code = 0;
3509
3510 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3511 if (!wqe)
3512 return -ENOMEM;
3513
3514 set_64bit_val(wqe, 16, hmc_fn_id);
3515 set_64bit_val(wqe, 32, commit_fpm_mem->pa);
3516
3517 hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |
3518 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |
3519 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3520
3521 dma_wmb(); /* make sure WQE is written before valid bit is set */
3522
3523 set_64bit_val(wqe, 24, hdr);
3524
3525 print_hex_dump_debug("WQE: COMMIT_FPM_VAL WQE", DUMP_PREFIX_OFFSET,
3526 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3527 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3528
3529 if (post_sq) {
3530 irdma_sc_cqp_post_sq(cqp);
3531 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3532 ret_code = irdma_cqp_poll_registers(cqp, tail,
3533 cqp->dev->hw_attrs.max_done_count);
3534 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3535 ret_code = irdma_sc_commit_fpm_val_done(cqp);
3536 }
3537
3538 return ret_code;
3539 }
3540
3541 /**
3542 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for
3543 * query fpm
3544 * @cqp: struct for cqp hw
3545 */
irdma_sc_query_fpm_val_done(struct irdma_sc_cqp * cqp)3546 static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)
3547 {
3548 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL,
3549 NULL);
3550 }
3551
3552 /**
3553 * irdma_sc_query_fpm_val - cqp wqe query fpm values
3554 * @cqp: struct for cqp hw
3555 * @scratch: u64 saved to be used during cqp completion
3556 * @hmc_fn_id: hmc function id
3557 * @query_fpm_mem: memory for return fpm values
3558 * @post_sq: flag for cqp db to ring
3559 * @wait_type: poll ccq or cqp registers for cqp completion
3560 */
irdma_sc_query_fpm_val(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,struct irdma_dma_mem * query_fpm_mem,bool post_sq,u8 wait_type)3561 static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
3562 u8 hmc_fn_id,
3563 struct irdma_dma_mem *query_fpm_mem,
3564 bool post_sq, u8 wait_type)
3565 {
3566 __le64 *wqe;
3567 u64 hdr;
3568 u32 tail, val, error;
3569 int ret_code = 0;
3570
3571 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3572 if (!wqe)
3573 return -ENOMEM;
3574
3575 set_64bit_val(wqe, 16, hmc_fn_id);
3576 set_64bit_val(wqe, 32, query_fpm_mem->pa);
3577
3578 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) |
3579 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3580 dma_wmb(); /* make sure WQE is written before valid bit is set */
3581
3582 set_64bit_val(wqe, 24, hdr);
3583
3584 print_hex_dump_debug("WQE: QUERY_FPM WQE", DUMP_PREFIX_OFFSET, 16, 8,
3585 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3586 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3587
3588 if (post_sq) {
3589 irdma_sc_cqp_post_sq(cqp);
3590 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3591 ret_code = irdma_cqp_poll_registers(cqp, tail,
3592 cqp->dev->hw_attrs.max_done_count);
3593 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3594 ret_code = irdma_sc_query_fpm_val_done(cqp);
3595 }
3596
3597 return ret_code;
3598 }
3599
3600 /**
3601 * irdma_sc_ceq_init - initialize ceq
3602 * @ceq: ceq sc structure
3603 * @info: ceq initialization info
3604 */
irdma_sc_ceq_init(struct irdma_sc_ceq * ceq,struct irdma_ceq_init_info * info)3605 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
3606 struct irdma_ceq_init_info *info)
3607 {
3608 u32 pble_obj_cnt;
3609
3610 if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||
3611 info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)
3612 return -EINVAL;
3613
3614 if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs)
3615 return -EINVAL;
3616 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3617
3618 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3619 return -EINVAL;
3620
3621 ceq->size = sizeof(*ceq);
3622 ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;
3623 ceq->ceq_id = info->ceq_id;
3624 ceq->dev = info->dev;
3625 ceq->elem_cnt = info->elem_cnt;
3626 ceq->ceq_elem_pa = info->ceqe_pa;
3627 ceq->virtual_map = info->virtual_map;
3628 ceq->itr_no_expire = info->itr_no_expire;
3629 ceq->reg_cq = info->reg_cq;
3630 ceq->reg_cq_size = 0;
3631 spin_lock_init(&ceq->req_cq_lock);
3632 ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
3633 ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
3634 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
3635 ceq->tph_en = info->tph_en;
3636 ceq->tph_val = info->tph_val;
3637 ceq->vsi = info->vsi;
3638 ceq->polarity = 1;
3639 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
3640 ceq->dev->ceq[info->ceq_id] = ceq;
3641
3642 return 0;
3643 }
3644
3645 /**
3646 * irdma_sc_ceq_create - create ceq wqe
3647 * @ceq: ceq sc structure
3648 * @scratch: u64 saved to be used during cqp completion
3649 * @post_sq: flag for cqp db to ring
3650 */
3651
irdma_sc_ceq_create(struct irdma_sc_ceq * ceq,u64 scratch,bool post_sq)3652 static int irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,
3653 bool post_sq)
3654 {
3655 struct irdma_sc_cqp *cqp;
3656 __le64 *wqe;
3657 u64 hdr;
3658
3659 cqp = ceq->dev->cqp;
3660 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3661 if (!wqe)
3662 return -ENOMEM;
3663 set_64bit_val(wqe, 16, ceq->elem_cnt);
3664 set_64bit_val(wqe, 32,
3665 (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
3666 set_64bit_val(wqe, 48,
3667 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
3668 set_64bit_val(wqe, 56,
3669 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |
3670 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));
3671 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |
3672 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) |
3673 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3674 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3675 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |
3676 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3677 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3678 dma_wmb(); /* make sure WQE is written before valid bit is set */
3679
3680 set_64bit_val(wqe, 24, hdr);
3681
3682 print_hex_dump_debug("WQE: CEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3683 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3684 if (post_sq)
3685 irdma_sc_cqp_post_sq(cqp);
3686
3687 return 0;
3688 }
3689
3690 /**
3691 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
3692 * @ceq: ceq sc structure
3693 */
irdma_sc_cceq_create_done(struct irdma_sc_ceq * ceq)3694 static int irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)
3695 {
3696 struct irdma_sc_cqp *cqp;
3697
3698 cqp = ceq->dev->cqp;
3699 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ,
3700 NULL);
3701 }
3702
3703 /**
3704 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
3705 * @ceq: ceq sc structure
3706 */
irdma_sc_cceq_destroy_done(struct irdma_sc_ceq * ceq)3707 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)
3708 {
3709 struct irdma_sc_cqp *cqp;
3710
3711 if (ceq->reg_cq)
3712 irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
3713
3714 cqp = ceq->dev->cqp;
3715 cqp->process_cqp_sds = irdma_update_sds_noccq;
3716
3717 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ,
3718 NULL);
3719 }
3720
3721 /**
3722 * irdma_sc_cceq_create - create cceq
3723 * @ceq: ceq sc structure
3724 * @scratch: u64 saved to be used during cqp completion
3725 */
irdma_sc_cceq_create(struct irdma_sc_ceq * ceq,u64 scratch)3726 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)
3727 {
3728 int ret_code;
3729 struct irdma_sc_dev *dev = ceq->dev;
3730
3731 dev->ccq->vsi = ceq->vsi;
3732 if (ceq->reg_cq) {
3733 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);
3734 if (ret_code)
3735 return ret_code;
3736 }
3737
3738 ret_code = irdma_sc_ceq_create(ceq, scratch, true);
3739 if (!ret_code)
3740 return irdma_sc_cceq_create_done(ceq);
3741
3742 return ret_code;
3743 }
3744
3745 /**
3746 * irdma_sc_ceq_destroy - destroy ceq
3747 * @ceq: ceq sc structure
3748 * @scratch: u64 saved to be used during cqp completion
3749 * @post_sq: flag for cqp db to ring
3750 */
irdma_sc_ceq_destroy(struct irdma_sc_ceq * ceq,u64 scratch,bool post_sq)3751 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq)
3752 {
3753 struct irdma_sc_cqp *cqp;
3754 __le64 *wqe;
3755 u64 hdr;
3756
3757 cqp = ceq->dev->cqp;
3758 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3759 if (!wqe)
3760 return -ENOMEM;
3761
3762 set_64bit_val(wqe, 16, ceq->elem_cnt);
3763 set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
3764 hdr = ceq->ceq_id |
3765 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) |
3766 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3767 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3768 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3769 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3770 dma_wmb(); /* make sure WQE is written before valid bit is set */
3771
3772 set_64bit_val(wqe, 24, hdr);
3773
3774 print_hex_dump_debug("WQE: CEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
3775 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3776 if (post_sq)
3777 irdma_sc_cqp_post_sq(cqp);
3778
3779 return 0;
3780 }
3781
3782 /**
3783 * irdma_sc_process_ceq - process ceq
3784 * @dev: sc device struct
3785 * @ceq: ceq sc structure
3786 *
3787 * It is expected caller serializes this function with cleanup_ceqes()
3788 * because these functions manipulate the same ceq
3789 */
irdma_sc_process_ceq(struct irdma_sc_dev * dev,struct irdma_sc_ceq * ceq)3790 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)
3791 {
3792 u64 temp;
3793 __le64 *ceqe;
3794 struct irdma_sc_cq *cq = NULL;
3795 struct irdma_sc_cq *temp_cq;
3796 u8 polarity;
3797 u32 cq_idx;
3798 unsigned long flags;
3799
3800 do {
3801 cq_idx = 0;
3802 ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);
3803 get_64bit_val(ceqe, 0, &temp);
3804 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3805 if (polarity != ceq->polarity)
3806 return NULL;
3807
3808 temp_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3809 if (!temp_cq) {
3810 cq_idx = IRDMA_INVALID_CQ_IDX;
3811 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3812
3813 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3814 ceq->polarity ^= 1;
3815 continue;
3816 }
3817
3818 cq = temp_cq;
3819 if (ceq->reg_cq) {
3820 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3821 cq_idx = irdma_sc_find_reg_cq(ceq, cq);
3822 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3823 }
3824
3825 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3826 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3827 ceq->polarity ^= 1;
3828 } while (cq_idx == IRDMA_INVALID_CQ_IDX);
3829
3830 if (cq)
3831 irdma_sc_cq_ack(cq);
3832 return cq;
3833 }
3834
3835 /**
3836 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq
3837 * @cq: cq for which the ceqes need to be cleaned up
3838 * @ceq: ceq ptr
3839 *
3840 * The function is called after the cq is destroyed to cleanup
3841 * its pending ceqe entries. It is expected caller serializes this
3842 * function with process_ceq() in interrupt context.
3843 */
irdma_sc_cleanup_ceqes(struct irdma_sc_cq * cq,struct irdma_sc_ceq * ceq)3844 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)
3845 {
3846 struct irdma_sc_cq *next_cq;
3847 u8 ceq_polarity = ceq->polarity;
3848 __le64 *ceqe;
3849 u8 polarity;
3850 u64 temp;
3851 int next;
3852 u32 i;
3853
3854 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);
3855
3856 for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) {
3857 ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);
3858
3859 get_64bit_val(ceqe, 0, &temp);
3860 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3861 if (polarity != ceq_polarity)
3862 return;
3863
3864 next_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3865 if (cq == next_cq)
3866 set_64bit_val(ceqe, 0, temp & IRDMA_CEQE_VALID);
3867
3868 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);
3869 if (!next)
3870 ceq_polarity ^= 1;
3871 }
3872 }
3873
3874 /**
3875 * irdma_sc_aeq_init - initialize aeq
3876 * @aeq: aeq structure ptr
3877 * @info: aeq initialization info
3878 */
irdma_sc_aeq_init(struct irdma_sc_aeq * aeq,struct irdma_aeq_init_info * info)3879 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
3880 struct irdma_aeq_init_info *info)
3881 {
3882 u32 pble_obj_cnt;
3883
3884 if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||
3885 info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)
3886 return -EINVAL;
3887
3888 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3889
3890 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3891 return -EINVAL;
3892
3893 aeq->size = sizeof(*aeq);
3894 aeq->polarity = 1;
3895 aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;
3896 aeq->dev = info->dev;
3897 aeq->elem_cnt = info->elem_cnt;
3898 aeq->aeq_elem_pa = info->aeq_elem_pa;
3899 IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
3900 aeq->virtual_map = info->virtual_map;
3901 aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
3902 aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
3903 aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
3904 aeq->msix_idx = info->msix_idx;
3905 info->dev->aeq = aeq;
3906
3907 return 0;
3908 }
3909
3910 /**
3911 * irdma_sc_aeq_create - create aeq
3912 * @aeq: aeq structure ptr
3913 * @scratch: u64 saved to be used during cqp completion
3914 * @post_sq: flag for cqp db to ring
3915 */
irdma_sc_aeq_create(struct irdma_sc_aeq * aeq,u64 scratch,bool post_sq)3916 static int irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch,
3917 bool post_sq)
3918 {
3919 __le64 *wqe;
3920 struct irdma_sc_cqp *cqp;
3921 u64 hdr;
3922
3923 cqp = aeq->dev->cqp;
3924 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3925 if (!wqe)
3926 return -ENOMEM;
3927 set_64bit_val(wqe, 16, aeq->elem_cnt);
3928 set_64bit_val(wqe, 32,
3929 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
3930 set_64bit_val(wqe, 48,
3931 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
3932
3933 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) |
3934 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
3935 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
3936 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3937 dma_wmb(); /* make sure WQE is written before valid bit is set */
3938
3939 set_64bit_val(wqe, 24, hdr);
3940
3941 print_hex_dump_debug("WQE: AEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3942 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3943 if (post_sq)
3944 irdma_sc_cqp_post_sq(cqp);
3945
3946 return 0;
3947 }
3948
3949 /**
3950 * irdma_sc_aeq_destroy - destroy aeq during close
3951 * @aeq: aeq structure ptr
3952 * @scratch: u64 saved to be used during cqp completion
3953 * @post_sq: flag for cqp db to ring
3954 */
irdma_sc_aeq_destroy(struct irdma_sc_aeq * aeq,u64 scratch,bool post_sq)3955 static int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch,
3956 bool post_sq)
3957 {
3958 __le64 *wqe;
3959 struct irdma_sc_cqp *cqp;
3960 struct irdma_sc_dev *dev;
3961 u64 hdr;
3962
3963 dev = aeq->dev;
3964 writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
3965
3966 cqp = dev->cqp;
3967 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3968 if (!wqe)
3969 return -ENOMEM;
3970 set_64bit_val(wqe, 16, aeq->elem_cnt);
3971 set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
3972 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) |
3973 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
3974 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
3975 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3976 dma_wmb(); /* make sure WQE is written before valid bit is set */
3977
3978 set_64bit_val(wqe, 24, hdr);
3979
3980 print_hex_dump_debug("WQE: AEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
3981 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3982 if (post_sq)
3983 irdma_sc_cqp_post_sq(cqp);
3984 return 0;
3985 }
3986
3987 /**
3988 * irdma_sc_get_next_aeqe - get next aeq entry
3989 * @aeq: aeq structure ptr
3990 * @info: aeqe info to be returned
3991 */
irdma_sc_get_next_aeqe(struct irdma_sc_aeq * aeq,struct irdma_aeqe_info * info)3992 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
3993 struct irdma_aeqe_info *info)
3994 {
3995 u64 temp, compl_ctx;
3996 __le64 *aeqe;
3997 u16 wqe_idx;
3998 u8 ae_src;
3999 u8 polarity;
4000
4001 aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
4002 get_64bit_val(aeqe, 8, &temp);
4003 polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
4004
4005 if (aeq->polarity != polarity)
4006 return -ENOENT;
4007
4008 /* Ensure AEQE contents are read after valid bit is checked */
4009 dma_rmb();
4010
4011 get_64bit_val(aeqe, 0, &compl_ctx);
4012
4013 print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8,
4014 aeqe, 16, false);
4015
4016 ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);
4017 wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp);
4018 info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |
4019 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18);
4020 info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);
4021 info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);
4022 info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);
4023 info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);
4024 info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp);
4025
4026 info->ae_src = ae_src;
4027 switch (info->ae_id) {
4028 case IRDMA_AE_PRIV_OPERATION_DENIED:
4029 case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:
4030 case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:
4031 case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:
4032 case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:
4033 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
4034 case IRDMA_AE_UDA_XMIT_BAD_PD:
4035 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
4036 case IRDMA_AE_BAD_CLOSE:
4037 case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:
4038 case IRDMA_AE_STAG_ZERO_INVALID:
4039 case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
4040 case IRDMA_AE_IB_INVALID_REQUEST:
4041 case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
4042 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
4043 case IRDMA_AE_IB_REMOTE_OP_ERROR:
4044 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
4045 case IRDMA_AE_DDP_UBE_INVALID_MO:
4046 case IRDMA_AE_DDP_UBE_INVALID_QN:
4047 case IRDMA_AE_DDP_NO_L_BIT:
4048 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4049 case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4050 case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
4051 case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
4052 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
4053 case IRDMA_AE_INVALID_ARP_ENTRY:
4054 case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
4055 case IRDMA_AE_STALE_ARP_ENTRY:
4056 case IRDMA_AE_INVALID_AH_ENTRY:
4057 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4058 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
4059 case IRDMA_AE_LLP_TOO_MANY_RETRIES:
4060 case IRDMA_AE_LLP_DOUBT_REACHABILITY:
4061 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
4062 case IRDMA_AE_RESET_SENT:
4063 case IRDMA_AE_TERMINATE_SENT:
4064 case IRDMA_AE_RESET_NOT_SENT:
4065 case IRDMA_AE_LCE_QP_CATASTROPHIC:
4066 case IRDMA_AE_QP_SUSPEND_COMPLETE:
4067 case IRDMA_AE_UDA_L4LEN_INVALID:
4068 info->qp = true;
4069 info->compl_ctx = compl_ctx;
4070 break;
4071 case IRDMA_AE_LCE_CQ_CATASTROPHIC:
4072 info->cq = true;
4073 info->compl_ctx = compl_ctx << 1;
4074 ae_src = IRDMA_AE_SOURCE_RSVD;
4075 break;
4076 case IRDMA_AE_ROCE_EMPTY_MCG:
4077 case IRDMA_AE_ROCE_BAD_MC_IP_ADDR:
4078 case IRDMA_AE_ROCE_BAD_MC_QPID:
4079 case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:
4080 fallthrough;
4081 case IRDMA_AE_LLP_CONNECTION_RESET:
4082 case IRDMA_AE_LLP_SYN_RECEIVED:
4083 case IRDMA_AE_LLP_FIN_RECEIVED:
4084 case IRDMA_AE_LLP_CLOSE_COMPLETE:
4085 case IRDMA_AE_LLP_TERMINATE_RECEIVED:
4086 case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
4087 ae_src = IRDMA_AE_SOURCE_RSVD;
4088 info->qp = true;
4089 info->compl_ctx = compl_ctx;
4090 break;
4091 default:
4092 break;
4093 }
4094
4095 switch (ae_src) {
4096 case IRDMA_AE_SOURCE_RQ:
4097 case IRDMA_AE_SOURCE_RQ_0011:
4098 info->qp = true;
4099 info->rq = true;
4100 info->wqe_idx = wqe_idx;
4101 info->compl_ctx = compl_ctx;
4102 break;
4103 case IRDMA_AE_SOURCE_CQ:
4104 case IRDMA_AE_SOURCE_CQ_0110:
4105 case IRDMA_AE_SOURCE_CQ_1010:
4106 case IRDMA_AE_SOURCE_CQ_1110:
4107 info->cq = true;
4108 info->compl_ctx = compl_ctx << 1;
4109 break;
4110 case IRDMA_AE_SOURCE_SQ:
4111 case IRDMA_AE_SOURCE_SQ_0111:
4112 info->qp = true;
4113 info->sq = true;
4114 info->wqe_idx = wqe_idx;
4115 info->compl_ctx = compl_ctx;
4116 break;
4117 case IRDMA_AE_SOURCE_IN_RR_WR:
4118 case IRDMA_AE_SOURCE_IN_RR_WR_1011:
4119 info->qp = true;
4120 info->compl_ctx = compl_ctx;
4121 info->in_rdrsp_wr = true;
4122 break;
4123 case IRDMA_AE_SOURCE_OUT_RR:
4124 case IRDMA_AE_SOURCE_OUT_RR_1111:
4125 info->qp = true;
4126 info->compl_ctx = compl_ctx;
4127 info->out_rdrsp = true;
4128 break;
4129 case IRDMA_AE_SOURCE_RSVD:
4130 default:
4131 break;
4132 }
4133
4134 IRDMA_RING_MOVE_TAIL(aeq->aeq_ring);
4135 if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring))
4136 aeq->polarity ^= 1;
4137
4138 return 0;
4139 }
4140
4141 /**
4142 * irdma_sc_repost_aeq_entries - repost completed aeq entries
4143 * @dev: sc device struct
4144 * @count: allocate count
4145 */
irdma_sc_repost_aeq_entries(struct irdma_sc_dev * dev,u32 count)4146 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)
4147 {
4148 writel(count, dev->hw_regs[IRDMA_AEQALLOC]);
4149 }
4150
4151 /**
4152 * irdma_sc_ccq_init - initialize control cq
4153 * @cq: sc's cq ctruct
4154 * @info: info for control cq initialization
4155 */
irdma_sc_ccq_init(struct irdma_sc_cq * cq,struct irdma_ccq_init_info * info)4156 int irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info)
4157 {
4158 u32 pble_obj_cnt;
4159
4160 if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||
4161 info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)
4162 return -EINVAL;
4163
4164 if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs)
4165 return -EINVAL;
4166
4167 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
4168
4169 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
4170 return -EINVAL;
4171
4172 cq->cq_pa = info->cq_pa;
4173 cq->cq_uk.cq_base = info->cq_base;
4174 cq->shadow_area_pa = info->shadow_area_pa;
4175 cq->cq_uk.shadow_area = info->shadow_area;
4176 cq->shadow_read_threshold = info->shadow_read_threshold;
4177 cq->dev = info->dev;
4178 cq->ceq_id = info->ceq_id;
4179 cq->cq_uk.cq_size = info->num_elem;
4180 cq->cq_type = IRDMA_CQ_TYPE_CQP;
4181 cq->ceqe_mask = info->ceqe_mask;
4182 IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
4183 cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
4184 cq->ceq_id_valid = info->ceq_id_valid;
4185 cq->tph_en = info->tph_en;
4186 cq->tph_val = info->tph_val;
4187 cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
4188 cq->pbl_list = info->pbl_list;
4189 cq->virtual_map = info->virtual_map;
4190 cq->pbl_chunk_size = info->pbl_chunk_size;
4191 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
4192 cq->cq_uk.polarity = true;
4193 cq->vsi = info->vsi;
4194 cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;
4195
4196 /* Only applicable to CQs other than CCQ so initialize to zero */
4197 cq->cq_uk.cqe_alloc_db = NULL;
4198
4199 info->dev->ccq = cq;
4200 return 0;
4201 }
4202
4203 /**
4204 * irdma_sc_ccq_create_done - poll cqp for ccq create
4205 * @ccq: ccq sc struct
4206 */
irdma_sc_ccq_create_done(struct irdma_sc_cq * ccq)4207 static inline int irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)
4208 {
4209 struct irdma_sc_cqp *cqp;
4210
4211 cqp = ccq->dev->cqp;
4212
4213 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL);
4214 }
4215
4216 /**
4217 * irdma_sc_ccq_create - create control cq
4218 * @ccq: ccq sc struct
4219 * @scratch: u64 saved to be used during cqp completion
4220 * @check_overflow: overlow flag for ccq
4221 * @post_sq: flag for cqp db to ring
4222 */
irdma_sc_ccq_create(struct irdma_sc_cq * ccq,u64 scratch,bool check_overflow,bool post_sq)4223 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
4224 bool check_overflow, bool post_sq)
4225 {
4226 int ret_code;
4227
4228 ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);
4229 if (ret_code)
4230 return ret_code;
4231
4232 if (post_sq) {
4233 ret_code = irdma_sc_ccq_create_done(ccq);
4234 if (ret_code)
4235 return ret_code;
4236 }
4237 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;
4238
4239 return 0;
4240 }
4241
4242 /**
4243 * irdma_sc_ccq_destroy - destroy ccq during close
4244 * @ccq: ccq sc struct
4245 * @scratch: u64 saved to be used during cqp completion
4246 * @post_sq: flag for cqp db to ring
4247 */
irdma_sc_ccq_destroy(struct irdma_sc_cq * ccq,u64 scratch,bool post_sq)4248 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq)
4249 {
4250 struct irdma_sc_cqp *cqp;
4251 __le64 *wqe;
4252 u64 hdr;
4253 int ret_code = 0;
4254 u32 tail, val, error;
4255
4256 cqp = ccq->dev->cqp;
4257 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4258 if (!wqe)
4259 return -ENOMEM;
4260
4261 set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
4262 set_64bit_val(wqe, 8, (uintptr_t)ccq >> 1);
4263 set_64bit_val(wqe, 40, ccq->shadow_area_pa);
4264
4265 hdr = ccq->cq_uk.cq_id |
4266 FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0),
4267 IRDMA_CQPSQ_CQ_CEQID) |
4268 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
4269 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) |
4270 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) |
4271 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) |
4272 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) |
4273 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4274 dma_wmb(); /* make sure WQE is written before valid bit is set */
4275
4276 set_64bit_val(wqe, 24, hdr);
4277
4278 print_hex_dump_debug("WQE: CCQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
4279 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4280 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4281
4282 if (post_sq) {
4283 irdma_sc_cqp_post_sq(cqp);
4284 ret_code = irdma_cqp_poll_registers(cqp, tail,
4285 cqp->dev->hw_attrs.max_done_count);
4286 }
4287
4288 cqp->process_cqp_sds = irdma_update_sds_noccq;
4289
4290 return ret_code;
4291 }
4292
4293 /**
4294 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
4295 * @dev : ptr to irdma_dev struct
4296 * @hmc_fn_id: hmc function id
4297 */
irdma_sc_init_iw_hmc(struct irdma_sc_dev * dev,u8 hmc_fn_id)4298 int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id)
4299 {
4300 struct irdma_hmc_info *hmc_info;
4301 struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4302 struct irdma_dma_mem query_fpm_mem;
4303 int ret_code = 0;
4304 u8 wait_type;
4305
4306 hmc_info = dev->hmc_info;
4307 hmc_fpm_misc = &dev->hmc_fpm_misc;
4308 query_fpm_mem.pa = dev->fpm_query_buf_pa;
4309 query_fpm_mem.va = dev->fpm_query_buf;
4310 hmc_info->hmc_fn_id = hmc_fn_id;
4311 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4312
4313 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4314 &query_fpm_mem, true, wait_type);
4315 if (ret_code)
4316 return ret_code;
4317
4318 /* parse the fpm_query_buf and fill hmc obj info */
4319 ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info,
4320 hmc_fpm_misc);
4321
4322 print_hex_dump_debug("HMC: QUERY FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4323 8, query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE,
4324 false);
4325 return ret_code;
4326 }
4327
4328 /**
4329 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp
4330 * command and populates fpm base address in hmc_info
4331 * @dev : ptr to irdma_dev struct
4332 * @hmc_fn_id: hmc function id
4333 */
irdma_sc_cfg_iw_fpm(struct irdma_sc_dev * dev,u8 hmc_fn_id)4334 static int irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id)
4335 {
4336 struct irdma_hmc_info *hmc_info;
4337 struct irdma_hmc_obj_info *obj_info;
4338 __le64 *buf;
4339 struct irdma_dma_mem commit_fpm_mem;
4340 int ret_code = 0;
4341 u8 wait_type;
4342
4343 hmc_info = dev->hmc_info;
4344 obj_info = hmc_info->hmc_obj;
4345 buf = dev->fpm_commit_buf;
4346
4347 set_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);
4348 set_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);
4349 set_64bit_val(buf, 16, (u64)0); /* RSRVD */
4350 set_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);
4351 set_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);
4352 set_64bit_val(buf, 40, (u64)0); /* RSVD */
4353 set_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);
4354 set_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);
4355 set_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);
4356 set_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);
4357 set_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);
4358 set_64bit_val(buf, 88,
4359 (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);
4360 set_64bit_val(buf, 96,
4361 (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);
4362 set_64bit_val(buf, 104,
4363 (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);
4364 set_64bit_val(buf, 112,
4365 (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);
4366 set_64bit_val(buf, 120, (u64)0); /* RSVD */
4367 set_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);
4368 set_64bit_val(buf, 136,
4369 (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);
4370 set_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);
4371 set_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);
4372 set_64bit_val(buf, 160,
4373 (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);
4374 set_64bit_val(buf, 168,
4375 (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);
4376
4377 commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
4378 commit_fpm_mem.va = dev->fpm_commit_buf;
4379
4380 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4381 print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4382 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4383 false);
4384 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4385 &commit_fpm_mem, true, wait_type);
4386 if (!ret_code)
4387 irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,
4388 hmc_info->hmc_obj,
4389 &hmc_info->sd_table.sd_cnt);
4390 print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4391 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4392 false);
4393
4394 return ret_code;
4395 }
4396
4397 /**
4398 * cqp_sds_wqe_fill - fill cqp wqe doe sd
4399 * @cqp: struct for cqp hw
4400 * @info: sd info for wqe
4401 * @scratch: u64 saved to be used during cqp completion
4402 */
cqp_sds_wqe_fill(struct irdma_sc_cqp * cqp,struct irdma_update_sds_info * info,u64 scratch)4403 static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp,
4404 struct irdma_update_sds_info *info, u64 scratch)
4405 {
4406 u64 data;
4407 u64 hdr;
4408 __le64 *wqe;
4409 int mem_entries, wqe_entries;
4410 struct irdma_dma_mem *sdbuf = &cqp->sdbuf;
4411 u64 offset = 0;
4412 u32 wqe_idx;
4413
4414 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
4415 if (!wqe)
4416 return -ENOMEM;
4417
4418 wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
4419 mem_entries = info->cnt - wqe_entries;
4420
4421 if (mem_entries) {
4422 offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;
4423 memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);
4424
4425 data = (u64)sdbuf->pa + offset;
4426 } else {
4427 data = 0;
4428 }
4429 data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id);
4430 set_64bit_val(wqe, 16, data);
4431
4432 switch (wqe_entries) {
4433 case 3:
4434 set_64bit_val(wqe, 48,
4435 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) |
4436 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4437
4438 set_64bit_val(wqe, 56, info->entry[2].data);
4439 fallthrough;
4440 case 2:
4441 set_64bit_val(wqe, 32,
4442 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) |
4443 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4444
4445 set_64bit_val(wqe, 40, info->entry[1].data);
4446 fallthrough;
4447 case 1:
4448 set_64bit_val(wqe, 0,
4449 FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd));
4450
4451 set_64bit_val(wqe, 8, info->entry[0].data);
4452 break;
4453 default:
4454 break;
4455 }
4456
4457 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) |
4458 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
4459 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries);
4460 dma_wmb(); /* make sure WQE is written before valid bit is set */
4461
4462 set_64bit_val(wqe, 24, hdr);
4463
4464 if (mem_entries)
4465 print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE Buffer",
4466 DUMP_PREFIX_OFFSET, 16, 8,
4467 (char *)sdbuf->va + offset,
4468 mem_entries << 4, false);
4469
4470 print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE", DUMP_PREFIX_OFFSET, 16,
4471 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4472
4473 return 0;
4474 }
4475
4476 /**
4477 * irdma_update_pe_sds - cqp wqe for sd
4478 * @dev: ptr to irdma_dev struct
4479 * @info: sd info for sd's
4480 * @scratch: u64 saved to be used during cqp completion
4481 */
irdma_update_pe_sds(struct irdma_sc_dev * dev,struct irdma_update_sds_info * info,u64 scratch)4482 static int irdma_update_pe_sds(struct irdma_sc_dev *dev,
4483 struct irdma_update_sds_info *info, u64 scratch)
4484 {
4485 struct irdma_sc_cqp *cqp = dev->cqp;
4486 int ret_code;
4487
4488 ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
4489 if (!ret_code)
4490 irdma_sc_cqp_post_sq(cqp);
4491
4492 return ret_code;
4493 }
4494
4495 /**
4496 * irdma_update_sds_noccq - update sd before ccq created
4497 * @dev: sc device struct
4498 * @info: sd info for sd's
4499 */
irdma_update_sds_noccq(struct irdma_sc_dev * dev,struct irdma_update_sds_info * info)4500 int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
4501 struct irdma_update_sds_info *info)
4502 {
4503 u32 error, val, tail;
4504 struct irdma_sc_cqp *cqp = dev->cqp;
4505 int ret_code;
4506
4507 ret_code = cqp_sds_wqe_fill(cqp, info, 0);
4508 if (ret_code)
4509 return ret_code;
4510
4511 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4512
4513 irdma_sc_cqp_post_sq(cqp);
4514 return irdma_cqp_poll_registers(cqp, tail,
4515 cqp->dev->hw_attrs.max_done_count);
4516 }
4517
4518 /**
4519 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
4520 * @cqp: struct for cqp hw
4521 * @scratch: u64 saved to be used during cqp completion
4522 * @hmc_fn_id: hmc function id
4523 * @post_sq: flag for cqp db to ring
4524 * @poll_registers: flag to poll register for cqp completion
4525 */
irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,bool post_sq,bool poll_registers)4526 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
4527 u8 hmc_fn_id, bool post_sq,
4528 bool poll_registers)
4529 {
4530 u64 hdr;
4531 __le64 *wqe;
4532 u32 tail, val, error;
4533
4534 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4535 if (!wqe)
4536 return -ENOMEM;
4537
4538 set_64bit_val(wqe, 16,
4539 FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id));
4540
4541 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
4542 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) |
4543 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4544 dma_wmb(); /* make sure WQE is written before valid bit is set */
4545
4546 set_64bit_val(wqe, 24, hdr);
4547
4548 print_hex_dump_debug("WQE: SHMC_PAGES_ALLOCATED WQE",
4549 DUMP_PREFIX_OFFSET, 16, 8, wqe,
4550 IRDMA_CQP_WQE_SIZE * 8, false);
4551 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4552
4553 if (post_sq) {
4554 irdma_sc_cqp_post_sq(cqp);
4555 if (poll_registers)
4556 /* check for cqp sq tail update */
4557 return irdma_cqp_poll_registers(cqp, tail,
4558 cqp->dev->hw_attrs.max_done_count);
4559 else
4560 return irdma_sc_poll_for_cqp_op_done(cqp,
4561 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,
4562 NULL);
4563 }
4564
4565 return 0;
4566 }
4567
4568 /**
4569 * irdma_cqp_ring_full - check if cqp ring is full
4570 * @cqp: struct for cqp hw
4571 */
irdma_cqp_ring_full(struct irdma_sc_cqp * cqp)4572 static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)
4573 {
4574 return IRDMA_RING_FULL_ERR(cqp->sq_ring);
4575 }
4576
4577 /**
4578 * irdma_est_sd - returns approximate number of SDs for HMC
4579 * @dev: sc device struct
4580 * @hmc_info: hmc structure, size and count for HMC objects
4581 */
irdma_est_sd(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info)4582 static u32 irdma_est_sd(struct irdma_sc_dev *dev,
4583 struct irdma_hmc_info *hmc_info)
4584 {
4585 int i;
4586 u64 size = 0;
4587 u64 sd;
4588
4589 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4590 if (i != IRDMA_HMC_IW_PBLE)
4591 size += round_up(hmc_info->hmc_obj[i].cnt *
4592 hmc_info->hmc_obj[i].size, 512);
4593 size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
4594 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);
4595 if (size & 0x1FFFFF)
4596 sd = (size >> 21) + 1; /* add 1 for remainder */
4597 else
4598 sd = size >> 21;
4599 if (sd > 0xFFFFFFFF) {
4600 ibdev_dbg(to_ibdev(dev), "HMC: sd overflow[%lld]\n", sd);
4601 sd = 0xFFFFFFFF - 1;
4602 }
4603
4604 return (u32)sd;
4605 }
4606
4607 /**
4608 * irdma_sc_query_rdma_features_done - poll cqp for query features done
4609 * @cqp: struct for cqp hw
4610 */
irdma_sc_query_rdma_features_done(struct irdma_sc_cqp * cqp)4611 static int irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp)
4612 {
4613 return irdma_sc_poll_for_cqp_op_done(cqp,
4614 IRDMA_CQP_OP_QUERY_RDMA_FEATURES,
4615 NULL);
4616 }
4617
4618 /**
4619 * irdma_sc_query_rdma_features - query RDMA features and FW ver
4620 * @cqp: struct for cqp hw
4621 * @buf: buffer to hold query info
4622 * @scratch: u64 saved to be used during cqp completion
4623 */
irdma_sc_query_rdma_features(struct irdma_sc_cqp * cqp,struct irdma_dma_mem * buf,u64 scratch)4624 static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
4625 struct irdma_dma_mem *buf, u64 scratch)
4626 {
4627 __le64 *wqe;
4628 u64 temp;
4629
4630 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4631 if (!wqe)
4632 return -ENOMEM;
4633
4634 temp = buf->pa;
4635 set_64bit_val(wqe, 32, temp);
4636
4637 temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID,
4638 cqp->polarity) |
4639 FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) |
4640 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES);
4641 dma_wmb(); /* make sure WQE is written before valid bit is set */
4642
4643 set_64bit_val(wqe, 24, temp);
4644
4645 print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4646 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4647 irdma_sc_cqp_post_sq(cqp);
4648
4649 return 0;
4650 }
4651
4652 /**
4653 * irdma_get_rdma_features - get RDMA features
4654 * @dev: sc device struct
4655 */
irdma_get_rdma_features(struct irdma_sc_dev * dev)4656 int irdma_get_rdma_features(struct irdma_sc_dev *dev)
4657 {
4658 int ret_code;
4659 struct irdma_dma_mem feat_buf;
4660 u64 temp;
4661 u16 byte_idx, feat_type, feat_cnt, feat_idx;
4662
4663 feat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE,
4664 IRDMA_FEATURE_BUF_ALIGNMENT);
4665 feat_buf.va = dma_alloc_coherent(dev->hw->device, feat_buf.size,
4666 &feat_buf.pa, GFP_KERNEL);
4667 if (!feat_buf.va)
4668 return -ENOMEM;
4669
4670 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4671 if (!ret_code)
4672 ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4673 if (ret_code)
4674 goto exit;
4675
4676 get_64bit_val(feat_buf.va, 0, &temp);
4677 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4678 if (feat_cnt < 2) {
4679 ret_code = -EINVAL;
4680 goto exit;
4681 } else if (feat_cnt > IRDMA_MAX_FEATURES) {
4682 ibdev_dbg(to_ibdev(dev),
4683 "DEV: feature buf size insufficient, retrying with larger buffer\n");
4684 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4685 feat_buf.pa);
4686 feat_buf.va = NULL;
4687 feat_buf.size = ALIGN(8 * feat_cnt,
4688 IRDMA_FEATURE_BUF_ALIGNMENT);
4689 feat_buf.va = dma_alloc_coherent(dev->hw->device,
4690 feat_buf.size, &feat_buf.pa,
4691 GFP_KERNEL);
4692 if (!feat_buf.va)
4693 return -ENOMEM;
4694
4695 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4696 if (!ret_code)
4697 ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4698 if (ret_code)
4699 goto exit;
4700
4701 get_64bit_val(feat_buf.va, 0, &temp);
4702 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4703 if (feat_cnt < 2) {
4704 ret_code = -EINVAL;
4705 goto exit;
4706 }
4707 }
4708
4709 print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4710 16, 8, feat_buf.va, feat_cnt * 8, false);
4711
4712 for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
4713 feat_idx++, byte_idx += 8) {
4714 get_64bit_val(feat_buf.va, byte_idx, &temp);
4715 feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
4716 if (feat_type >= IRDMA_MAX_FEATURES) {
4717 ibdev_dbg(to_ibdev(dev),
4718 "DEV: found unrecognized feature type %d\n",
4719 feat_type);
4720 continue;
4721 }
4722 dev->feature_info[feat_type] = temp;
4723 }
4724 exit:
4725 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4726 feat_buf.pa);
4727 feat_buf.va = NULL;
4728 return ret_code;
4729 }
4730
irdma_q1_cnt(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4731 static u32 irdma_q1_cnt(struct irdma_sc_dev *dev,
4732 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4733 {
4734 u32 q1_cnt;
4735
4736 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
4737 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted);
4738 } else {
4739 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
4740 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512);
4741 else
4742 q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted;
4743 }
4744
4745 return q1_cnt;
4746 }
4747
cfg_fpm_value_gen_1(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4748 static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev,
4749 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4750 {
4751 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes);
4752 }
4753
cfg_fpm_value_gen_2(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4754 static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev,
4755 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4756 {
4757 struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;
4758
4759 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =
4760 4 * hmc_fpm_misc->xf_block_size * qpwanted;
4761
4762 hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;
4763
4764 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)
4765 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;
4766 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)
4767 hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =
4768 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /
4769 hmc_fpm_misc->rrf_block_size;
4770 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)
4771 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;
4772 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)
4773 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =
4774 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /
4775 hmc_fpm_misc->ooiscf_block_size;
4776 }
4777
4778 /**
4779 * irdma_cfg_fpm_val - configure HMC objects
4780 * @dev: sc device struct
4781 * @qp_count: desired qp count
4782 */
irdma_cfg_fpm_val(struct irdma_sc_dev * dev,u32 qp_count)4783 int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)
4784 {
4785 struct irdma_virt_mem virt_mem;
4786 u32 i, mem_size;
4787 u32 qpwanted, mrwanted, pblewanted;
4788 u32 powerof2, hte;
4789 u32 sd_needed;
4790 u32 sd_diff;
4791 u32 loop_count = 0;
4792 struct irdma_hmc_info *hmc_info;
4793 struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4794 int ret_code = 0;
4795
4796 hmc_info = dev->hmc_info;
4797 hmc_fpm_misc = &dev->hmc_fpm_misc;
4798
4799 ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);
4800 if (ret_code) {
4801 ibdev_dbg(to_ibdev(dev),
4802 "HMC: irdma_sc_init_iw_hmc returned error_code = %d\n",
4803 ret_code);
4804 return ret_code;
4805 }
4806
4807 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4808 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
4809 sd_needed = irdma_est_sd(dev, hmc_info);
4810 ibdev_dbg(to_ibdev(dev),
4811 "HMC: FW max resources sd_needed[%08d] first_sd_index[%04d]\n",
4812 sd_needed, hmc_info->first_sd_index);
4813 ibdev_dbg(to_ibdev(dev), "HMC: sd count %d where max sd is %d\n",
4814 hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds);
4815
4816 qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);
4817
4818 powerof2 = 1;
4819 while (powerof2 <= qpwanted)
4820 powerof2 *= 2;
4821 powerof2 /= 2;
4822 qpwanted = powerof2;
4823
4824 mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;
4825 pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;
4826
4827 ibdev_dbg(to_ibdev(dev),
4828 "HMC: req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n",
4829 qp_count, hmc_fpm_misc->max_sds,
4830 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,
4831 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,
4832 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,
4833 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,
4834 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,
4835 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);
4836 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =
4837 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;
4838 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =
4839 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;
4840 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =
4841 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;
4842
4843 hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
4844
4845 while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)
4846 qpwanted /= 2;
4847
4848 do {
4849 ++loop_count;
4850 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;
4851 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =
4852 min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);
4853 hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */
4854 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;
4855
4856 hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);
4857 powerof2 = 1;
4858 while (powerof2 < hte)
4859 powerof2 *= 2;
4860 hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =
4861 powerof2 * hmc_fpm_misc->ht_multiplier;
4862 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
4863 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted);
4864 else
4865 cfg_fpm_value_gen_2(dev, hmc_info, qpwanted);
4866
4867 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted);
4868 hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =
4869 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
4870 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =
4871 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
4872 hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =
4873 (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;
4874
4875 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4876 sd_needed = irdma_est_sd(dev, hmc_info);
4877 ibdev_dbg(to_ibdev(dev),
4878 "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",
4879 sd_needed, hmc_fpm_misc->max_sds, mrwanted,
4880 pblewanted, qpwanted);
4881
4882 /* Do not reduce resources further. All objects fit with max SDs */
4883 if (sd_needed <= hmc_fpm_misc->max_sds)
4884 break;
4885
4886 sd_diff = sd_needed - hmc_fpm_misc->max_sds;
4887 if (sd_diff > 128) {
4888 if (!(loop_count % 2) && qpwanted > 128) {
4889 qpwanted /= 2;
4890 } else {
4891 mrwanted /= 2;
4892 pblewanted /= 2;
4893 }
4894 continue;
4895 }
4896 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&
4897 pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {
4898 pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;
4899 continue;
4900 } else if (pblewanted > (100 * FPM_MULTIPLIER)) {
4901 pblewanted -= 10 * FPM_MULTIPLIER;
4902 } else if (pblewanted > FPM_MULTIPLIER) {
4903 pblewanted -= FPM_MULTIPLIER;
4904 } else if (qpwanted <= 128) {
4905 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)
4906 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;
4907 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4908 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4909 }
4910 if (mrwanted > FPM_MULTIPLIER)
4911 mrwanted -= FPM_MULTIPLIER;
4912 if (!(loop_count % 10) && qpwanted > 128) {
4913 qpwanted /= 2;
4914 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4915 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4916 }
4917 } while (loop_count < 2000);
4918
4919 if (sd_needed > hmc_fpm_misc->max_sds) {
4920 ibdev_dbg(to_ibdev(dev),
4921 "HMC: cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n",
4922 loop_count, sd_needed, hmc_info->sd_table.sd_cnt);
4923 return -EINVAL;
4924 }
4925
4926 if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {
4927 pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 *
4928 FPM_MULTIPLIER;
4929 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4930 sd_needed = irdma_est_sd(dev, hmc_info);
4931 }
4932
4933 ibdev_dbg(to_ibdev(dev),
4934 "HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n",
4935 loop_count, sd_needed,
4936 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,
4937 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,
4938 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,
4939 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,
4940 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,
4941 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,
4942 hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index);
4943
4944 ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);
4945 if (ret_code) {
4946 ibdev_dbg(to_ibdev(dev),
4947 "HMC: cfg_iw_fpm returned error_code[x%08X]\n",
4948 readl(dev->hw_regs[IRDMA_CQPERRCODES]));
4949 return ret_code;
4950 }
4951
4952 mem_size = sizeof(struct irdma_hmc_sd_entry) *
4953 (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
4954 virt_mem.size = mem_size;
4955 virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL);
4956 if (!virt_mem.va) {
4957 ibdev_dbg(to_ibdev(dev),
4958 "HMC: failed to allocate memory for sd_entry buffer\n");
4959 return -ENOMEM;
4960 }
4961 hmc_info->sd_table.sd_entry = virt_mem.va;
4962
4963 return ret_code;
4964 }
4965
4966 /**
4967 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
4968 * @dev: rdma device
4969 * @pcmdinfo: cqp command info
4970 */
irdma_exec_cqp_cmd(struct irdma_sc_dev * dev,struct cqp_cmds_info * pcmdinfo)4971 static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
4972 struct cqp_cmds_info *pcmdinfo)
4973 {
4974 int status;
4975 struct irdma_dma_mem val_mem;
4976 bool alloc = false;
4977
4978 dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
4979 switch (pcmdinfo->cqp_cmd) {
4980 case IRDMA_OP_CEQ_DESTROY:
4981 status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
4982 pcmdinfo->in.u.ceq_destroy.scratch,
4983 pcmdinfo->post_sq);
4984 break;
4985 case IRDMA_OP_AEQ_DESTROY:
4986 status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
4987 pcmdinfo->in.u.aeq_destroy.scratch,
4988 pcmdinfo->post_sq);
4989
4990 break;
4991 case IRDMA_OP_CEQ_CREATE:
4992 status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
4993 pcmdinfo->in.u.ceq_create.scratch,
4994 pcmdinfo->post_sq);
4995 break;
4996 case IRDMA_OP_AEQ_CREATE:
4997 status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
4998 pcmdinfo->in.u.aeq_create.scratch,
4999 pcmdinfo->post_sq);
5000 break;
5001 case IRDMA_OP_QP_UPLOAD_CONTEXT:
5002 status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,
5003 &pcmdinfo->in.u.qp_upload_context.info,
5004 pcmdinfo->in.u.qp_upload_context.scratch,
5005 pcmdinfo->post_sq);
5006 break;
5007 case IRDMA_OP_CQ_CREATE:
5008 status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,
5009 pcmdinfo->in.u.cq_create.scratch,
5010 pcmdinfo->in.u.cq_create.check_overflow,
5011 pcmdinfo->post_sq);
5012 break;
5013 case IRDMA_OP_CQ_MODIFY:
5014 status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq,
5015 &pcmdinfo->in.u.cq_modify.info,
5016 pcmdinfo->in.u.cq_modify.scratch,
5017 pcmdinfo->post_sq);
5018 break;
5019 case IRDMA_OP_CQ_DESTROY:
5020 status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,
5021 pcmdinfo->in.u.cq_destroy.scratch,
5022 pcmdinfo->post_sq);
5023 break;
5024 case IRDMA_OP_QP_FLUSH_WQES:
5025 status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
5026 &pcmdinfo->in.u.qp_flush_wqes.info,
5027 pcmdinfo->in.u.qp_flush_wqes.scratch,
5028 pcmdinfo->post_sq);
5029 break;
5030 case IRDMA_OP_GEN_AE:
5031 status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
5032 &pcmdinfo->in.u.gen_ae.info,
5033 pcmdinfo->in.u.gen_ae.scratch,
5034 pcmdinfo->post_sq);
5035 break;
5036 case IRDMA_OP_MANAGE_PUSH_PAGE:
5037 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,
5038 &pcmdinfo->in.u.manage_push_page.info,
5039 pcmdinfo->in.u.manage_push_page.scratch,
5040 pcmdinfo->post_sq);
5041 break;
5042 case IRDMA_OP_UPDATE_PE_SDS:
5043 status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,
5044 &pcmdinfo->in.u.update_pe_sds.info,
5045 pcmdinfo->in.u.update_pe_sds.scratch);
5046 break;
5047 case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:
5048 /* switch to calling through the call table */
5049 status =
5050 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
5051 &pcmdinfo->in.u.manage_hmc_pm.info,
5052 pcmdinfo->in.u.manage_hmc_pm.scratch,
5053 true);
5054 break;
5055 case IRDMA_OP_SUSPEND:
5056 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,
5057 pcmdinfo->in.u.suspend_resume.qp,
5058 pcmdinfo->in.u.suspend_resume.scratch);
5059 break;
5060 case IRDMA_OP_RESUME:
5061 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,
5062 pcmdinfo->in.u.suspend_resume.qp,
5063 pcmdinfo->in.u.suspend_resume.scratch);
5064 break;
5065 case IRDMA_OP_QUERY_FPM_VAL:
5066 val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;
5067 val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;
5068 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,
5069 pcmdinfo->in.u.query_fpm_val.scratch,
5070 pcmdinfo->in.u.query_fpm_val.hmc_fn_id,
5071 &val_mem, true, IRDMA_CQP_WAIT_EVENT);
5072 break;
5073 case IRDMA_OP_COMMIT_FPM_VAL:
5074 val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;
5075 val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;
5076 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,
5077 pcmdinfo->in.u.commit_fpm_val.scratch,
5078 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,
5079 &val_mem,
5080 true,
5081 IRDMA_CQP_WAIT_EVENT);
5082 break;
5083 case IRDMA_OP_STATS_ALLOCATE:
5084 alloc = true;
5085 fallthrough;
5086 case IRDMA_OP_STATS_FREE:
5087 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,
5088 &pcmdinfo->in.u.stats_manage.info,
5089 alloc,
5090 pcmdinfo->in.u.stats_manage.scratch);
5091 break;
5092 case IRDMA_OP_STATS_GATHER:
5093 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,
5094 &pcmdinfo->in.u.stats_gather.info,
5095 pcmdinfo->in.u.stats_gather.scratch);
5096 break;
5097 case IRDMA_OP_WS_MODIFY_NODE:
5098 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5099 &pcmdinfo->in.u.ws_node.info,
5100 IRDMA_MODIFY_NODE,
5101 pcmdinfo->in.u.ws_node.scratch);
5102 break;
5103 case IRDMA_OP_WS_DELETE_NODE:
5104 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5105 &pcmdinfo->in.u.ws_node.info,
5106 IRDMA_DEL_NODE,
5107 pcmdinfo->in.u.ws_node.scratch);
5108 break;
5109 case IRDMA_OP_WS_ADD_NODE:
5110 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5111 &pcmdinfo->in.u.ws_node.info,
5112 IRDMA_ADD_NODE,
5113 pcmdinfo->in.u.ws_node.scratch);
5114 break;
5115 case IRDMA_OP_SET_UP_MAP:
5116 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,
5117 &pcmdinfo->in.u.up_map.info,
5118 pcmdinfo->in.u.up_map.scratch);
5119 break;
5120 case IRDMA_OP_QUERY_RDMA_FEATURES:
5121 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,
5122 &pcmdinfo->in.u.query_rdma.query_buff_mem,
5123 pcmdinfo->in.u.query_rdma.scratch);
5124 break;
5125 case IRDMA_OP_DELETE_ARP_CACHE_ENTRY:
5126 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,
5127 pcmdinfo->in.u.del_arp_cache_entry.scratch,
5128 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
5129 pcmdinfo->post_sq);
5130 break;
5131 case IRDMA_OP_MANAGE_APBVT_ENTRY:
5132 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,
5133 &pcmdinfo->in.u.manage_apbvt_entry.info,
5134 pcmdinfo->in.u.manage_apbvt_entry.scratch,
5135 pcmdinfo->post_sq);
5136 break;
5137 case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:
5138 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,
5139 &pcmdinfo->in.u.manage_qhash_table_entry.info,
5140 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
5141 pcmdinfo->post_sq);
5142 break;
5143 case IRDMA_OP_QP_MODIFY:
5144 status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,
5145 &pcmdinfo->in.u.qp_modify.info,
5146 pcmdinfo->in.u.qp_modify.scratch,
5147 pcmdinfo->post_sq);
5148 break;
5149 case IRDMA_OP_QP_CREATE:
5150 status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,
5151 &pcmdinfo->in.u.qp_create.info,
5152 pcmdinfo->in.u.qp_create.scratch,
5153 pcmdinfo->post_sq);
5154 break;
5155 case IRDMA_OP_QP_DESTROY:
5156 status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,
5157 pcmdinfo->in.u.qp_destroy.scratch,
5158 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
5159 pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,
5160 pcmdinfo->post_sq);
5161 break;
5162 case IRDMA_OP_ALLOC_STAG:
5163 status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,
5164 &pcmdinfo->in.u.alloc_stag.info,
5165 pcmdinfo->in.u.alloc_stag.scratch,
5166 pcmdinfo->post_sq);
5167 break;
5168 case IRDMA_OP_MR_REG_NON_SHARED:
5169 status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,
5170 &pcmdinfo->in.u.mr_reg_non_shared.info,
5171 pcmdinfo->in.u.mr_reg_non_shared.scratch,
5172 pcmdinfo->post_sq);
5173 break;
5174 case IRDMA_OP_DEALLOC_STAG:
5175 status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,
5176 &pcmdinfo->in.u.dealloc_stag.info,
5177 pcmdinfo->in.u.dealloc_stag.scratch,
5178 pcmdinfo->post_sq);
5179 break;
5180 case IRDMA_OP_MW_ALLOC:
5181 status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,
5182 &pcmdinfo->in.u.mw_alloc.info,
5183 pcmdinfo->in.u.mw_alloc.scratch,
5184 pcmdinfo->post_sq);
5185 break;
5186 case IRDMA_OP_ADD_ARP_CACHE_ENTRY:
5187 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,
5188 &pcmdinfo->in.u.add_arp_cache_entry.info,
5189 pcmdinfo->in.u.add_arp_cache_entry.scratch,
5190 pcmdinfo->post_sq);
5191 break;
5192 case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:
5193 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,
5194 pcmdinfo->in.u.alloc_local_mac_entry.scratch,
5195 pcmdinfo->post_sq);
5196 break;
5197 case IRDMA_OP_ADD_LOCAL_MAC_ENTRY:
5198 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,
5199 &pcmdinfo->in.u.add_local_mac_entry.info,
5200 pcmdinfo->in.u.add_local_mac_entry.scratch,
5201 pcmdinfo->post_sq);
5202 break;
5203 case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:
5204 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,
5205 pcmdinfo->in.u.del_local_mac_entry.scratch,
5206 pcmdinfo->in.u.del_local_mac_entry.entry_idx,
5207 pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,
5208 pcmdinfo->post_sq);
5209 break;
5210 case IRDMA_OP_AH_CREATE:
5211 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp,
5212 &pcmdinfo->in.u.ah_create.info,
5213 pcmdinfo->in.u.ah_create.scratch);
5214 break;
5215 case IRDMA_OP_AH_DESTROY:
5216 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,
5217 &pcmdinfo->in.u.ah_destroy.info,
5218 pcmdinfo->in.u.ah_destroy.scratch);
5219 break;
5220 case IRDMA_OP_MC_CREATE:
5221 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp,
5222 &pcmdinfo->in.u.mc_create.info,
5223 pcmdinfo->in.u.mc_create.scratch);
5224 break;
5225 case IRDMA_OP_MC_DESTROY:
5226 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp,
5227 &pcmdinfo->in.u.mc_destroy.info,
5228 pcmdinfo->in.u.mc_destroy.scratch);
5229 break;
5230 case IRDMA_OP_MC_MODIFY:
5231 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp,
5232 &pcmdinfo->in.u.mc_modify.info,
5233 pcmdinfo->in.u.mc_modify.scratch);
5234 break;
5235 default:
5236 status = -EOPNOTSUPP;
5237 break;
5238 }
5239
5240 return status;
5241 }
5242
5243 /**
5244 * irdma_process_cqp_cmd - process all cqp commands
5245 * @dev: sc device struct
5246 * @pcmdinfo: cqp command info
5247 */
irdma_process_cqp_cmd(struct irdma_sc_dev * dev,struct cqp_cmds_info * pcmdinfo)5248 int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
5249 struct cqp_cmds_info *pcmdinfo)
5250 {
5251 int status = 0;
5252 unsigned long flags;
5253
5254 spin_lock_irqsave(&dev->cqp_lock, flags);
5255 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))
5256 status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5257 else
5258 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
5259 spin_unlock_irqrestore(&dev->cqp_lock, flags);
5260 return status;
5261 }
5262
5263 /**
5264 * irdma_process_bh - called from tasklet for cqp list
5265 * @dev: sc device struct
5266 */
irdma_process_bh(struct irdma_sc_dev * dev)5267 int irdma_process_bh(struct irdma_sc_dev *dev)
5268 {
5269 int status = 0;
5270 struct cqp_cmds_info *pcmdinfo;
5271 unsigned long flags;
5272
5273 spin_lock_irqsave(&dev->cqp_lock, flags);
5274 while (!list_empty(&dev->cqp_cmd_head) &&
5275 !irdma_cqp_ring_full(dev->cqp)) {
5276 pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev);
5277 status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5278 if (status)
5279 break;
5280 }
5281 spin_unlock_irqrestore(&dev->cqp_lock, flags);
5282 return status;
5283 }
5284
5285 /**
5286 * irdma_cfg_aeq- Configure AEQ interrupt
5287 * @dev: pointer to the device structure
5288 * @idx: vector index
5289 * @enable: True to enable, False disables
5290 */
irdma_cfg_aeq(struct irdma_sc_dev * dev,u32 idx,bool enable)5291 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
5292 {
5293 u32 reg_val;
5294
5295 reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
5296 FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
5297 FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, 3);
5298 writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
5299 }
5300
5301 /**
5302 * sc_vsi_update_stats - Update statistics
5303 * @vsi: sc_vsi instance to update
5304 */
sc_vsi_update_stats(struct irdma_sc_vsi * vsi)5305 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
5306 {
5307 struct irdma_gather_stats *gather_stats;
5308 struct irdma_gather_stats *last_gather_stats;
5309
5310 gather_stats = vsi->pestat->gather_info.gather_stats_va;
5311 last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
5312 irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
5313 last_gather_stats);
5314 }
5315
5316 /**
5317 * irdma_wait_pe_ready - Check if firmware is ready
5318 * @dev: provides access to registers
5319 */
irdma_wait_pe_ready(struct irdma_sc_dev * dev)5320 static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)
5321 {
5322 u32 statuscpu0;
5323 u32 statuscpu1;
5324 u32 statuscpu2;
5325 u32 retrycount = 0;
5326
5327 do {
5328 statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);
5329 statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);
5330 statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);
5331 if (statuscpu0 == 0x80 && statuscpu1 == 0x80 &&
5332 statuscpu2 == 0x80)
5333 return 0;
5334 mdelay(1000);
5335 } while (retrycount++ < dev->hw_attrs.max_pe_ready_count);
5336 return -1;
5337 }
5338
irdma_sc_init_hw(struct irdma_sc_dev * dev)5339 static inline void irdma_sc_init_hw(struct irdma_sc_dev *dev)
5340 {
5341 switch (dev->hw_attrs.uk_attrs.hw_rev) {
5342 case IRDMA_GEN_1:
5343 i40iw_init_hw(dev);
5344 break;
5345 case IRDMA_GEN_2:
5346 icrdma_init_hw(dev);
5347 break;
5348 }
5349 }
5350
5351 /**
5352 * irdma_sc_dev_init - Initialize control part of device
5353 * @ver: version
5354 * @dev: Device pointer
5355 * @info: Device init info
5356 */
irdma_sc_dev_init(enum irdma_vers ver,struct irdma_sc_dev * dev,struct irdma_device_init_info * info)5357 int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
5358 struct irdma_device_init_info *info)
5359 {
5360 u32 val;
5361 int ret_code = 0;
5362 u8 db_size;
5363
5364 INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */
5365 mutex_init(&dev->ws_mutex);
5366 dev->hmc_fn_id = info->hmc_fn_id;
5367 dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5368 dev->fpm_query_buf = info->fpm_query_buf;
5369 dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5370 dev->fpm_commit_buf = info->fpm_commit_buf;
5371 dev->hw = info->hw;
5372 dev->hw->hw_addr = info->bar0;
5373 /* Setup the hardware limits, hmc may limit further */
5374 dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;
5375 dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;
5376 dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;
5377 dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;
5378 dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;
5379 dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;
5380 dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;
5381 dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;
5382 dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;
5383 dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
5384 dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
5385 dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
5386 dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
5387 dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
5388 dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
5389 dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
5390
5391 dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;
5392 dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;
5393 dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;
5394 dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;
5395
5396 dev->hw_attrs.max_pe_ready_count = 14;
5397 dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;
5398 dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;
5399 dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;
5400
5401 dev->hw_attrs.uk_attrs.hw_rev = ver;
5402 irdma_sc_init_hw(dev);
5403
5404 if (irdma_wait_pe_ready(dev))
5405 return -ETIMEDOUT;
5406
5407 val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);
5408 db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val);
5409 if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) {
5410 ibdev_dbg(to_ibdev(dev),
5411 "DEV: RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n",
5412 val, db_size);
5413 return -ENODEV;
5414 }
5415 dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
5416
5417 return ret_code;
5418 }
5419
5420 /**
5421 * irdma_update_stats - Update statistics
5422 * @hw_stats: hw_stats instance to update
5423 * @gather_stats: updated stat counters
5424 * @last_gather_stats: last stat counters
5425 */
irdma_update_stats(struct irdma_dev_hw_stats * hw_stats,struct irdma_gather_stats * gather_stats,struct irdma_gather_stats * last_gather_stats)5426 void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
5427 struct irdma_gather_stats *gather_stats,
5428 struct irdma_gather_stats *last_gather_stats)
5429 {
5430 u64 *stats_val = hw_stats->stats_val_32;
5431
5432 stats_val[IRDMA_HW_STAT_INDEX_RXVLANERR] +=
5433 IRDMA_STATS_DELTA(gather_stats->rxvlanerr,
5434 last_gather_stats->rxvlanerr,
5435 IRDMA_MAX_STATS_32);
5436 stats_val[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] +=
5437 IRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,
5438 last_gather_stats->ip4rxdiscard,
5439 IRDMA_MAX_STATS_32);
5440 stats_val[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] +=
5441 IRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,
5442 last_gather_stats->ip4rxtrunc,
5443 IRDMA_MAX_STATS_32);
5444 stats_val[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] +=
5445 IRDMA_STATS_DELTA(gather_stats->ip4txnoroute,
5446 last_gather_stats->ip4txnoroute,
5447 IRDMA_MAX_STATS_32);
5448 stats_val[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] +=
5449 IRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,
5450 last_gather_stats->ip6rxdiscard,
5451 IRDMA_MAX_STATS_32);
5452 stats_val[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] +=
5453 IRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,
5454 last_gather_stats->ip6rxtrunc,
5455 IRDMA_MAX_STATS_32);
5456 stats_val[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] +=
5457 IRDMA_STATS_DELTA(gather_stats->ip6txnoroute,
5458 last_gather_stats->ip6txnoroute,
5459 IRDMA_MAX_STATS_32);
5460 stats_val[IRDMA_HW_STAT_INDEX_TCPRTXSEG] +=
5461 IRDMA_STATS_DELTA(gather_stats->tcprtxseg,
5462 last_gather_stats->tcprtxseg,
5463 IRDMA_MAX_STATS_32);
5464 stats_val[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] +=
5465 IRDMA_STATS_DELTA(gather_stats->tcprxopterr,
5466 last_gather_stats->tcprxopterr,
5467 IRDMA_MAX_STATS_32);
5468 stats_val[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] +=
5469 IRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,
5470 last_gather_stats->tcprxprotoerr,
5471 IRDMA_MAX_STATS_32);
5472 stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] +=
5473 IRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,
5474 last_gather_stats->rxrpcnphandled,
5475 IRDMA_MAX_STATS_32);
5476 stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] +=
5477 IRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,
5478 last_gather_stats->rxrpcnpignored,
5479 IRDMA_MAX_STATS_32);
5480 stats_val[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] +=
5481 IRDMA_STATS_DELTA(gather_stats->txnpcnpsent,
5482 last_gather_stats->txnpcnpsent,
5483 IRDMA_MAX_STATS_32);
5484 stats_val = hw_stats->stats_val_64;
5485 stats_val[IRDMA_HW_STAT_INDEX_IP4RXOCTS] +=
5486 IRDMA_STATS_DELTA(gather_stats->ip4rxocts,
5487 last_gather_stats->ip4rxocts,
5488 IRDMA_MAX_STATS_48);
5489 stats_val[IRDMA_HW_STAT_INDEX_IP4RXPKTS] +=
5490 IRDMA_STATS_DELTA(gather_stats->ip4rxpkts,
5491 last_gather_stats->ip4rxpkts,
5492 IRDMA_MAX_STATS_48);
5493 stats_val[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] +=
5494 IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
5495 last_gather_stats->ip4txfrag,
5496 IRDMA_MAX_STATS_48);
5497 stats_val[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] +=
5498 IRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,
5499 last_gather_stats->ip4rxmcpkts,
5500 IRDMA_MAX_STATS_48);
5501 stats_val[IRDMA_HW_STAT_INDEX_IP4TXOCTS] +=
5502 IRDMA_STATS_DELTA(gather_stats->ip4txocts,
5503 last_gather_stats->ip4txocts,
5504 IRDMA_MAX_STATS_48);
5505 stats_val[IRDMA_HW_STAT_INDEX_IP4TXPKTS] +=
5506 IRDMA_STATS_DELTA(gather_stats->ip4txpkts,
5507 last_gather_stats->ip4txpkts,
5508 IRDMA_MAX_STATS_48);
5509 stats_val[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] +=
5510 IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
5511 last_gather_stats->ip4txfrag,
5512 IRDMA_MAX_STATS_48);
5513 stats_val[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] +=
5514 IRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,
5515 last_gather_stats->ip4txmcpkts,
5516 IRDMA_MAX_STATS_48);
5517 stats_val[IRDMA_HW_STAT_INDEX_IP6RXOCTS] +=
5518 IRDMA_STATS_DELTA(gather_stats->ip6rxocts,
5519 last_gather_stats->ip6rxocts,
5520 IRDMA_MAX_STATS_48);
5521 stats_val[IRDMA_HW_STAT_INDEX_IP6RXPKTS] +=
5522 IRDMA_STATS_DELTA(gather_stats->ip6rxpkts,
5523 last_gather_stats->ip6rxpkts,
5524 IRDMA_MAX_STATS_48);
5525 stats_val[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] +=
5526 IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
5527 last_gather_stats->ip6txfrags,
5528 IRDMA_MAX_STATS_48);
5529 stats_val[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] +=
5530 IRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,
5531 last_gather_stats->ip6rxmcpkts,
5532 IRDMA_MAX_STATS_48);
5533 stats_val[IRDMA_HW_STAT_INDEX_IP6TXOCTS] +=
5534 IRDMA_STATS_DELTA(gather_stats->ip6txocts,
5535 last_gather_stats->ip6txocts,
5536 IRDMA_MAX_STATS_48);
5537 stats_val[IRDMA_HW_STAT_INDEX_IP6TXPKTS] +=
5538 IRDMA_STATS_DELTA(gather_stats->ip6txpkts,
5539 last_gather_stats->ip6txpkts,
5540 IRDMA_MAX_STATS_48);
5541 stats_val[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] +=
5542 IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
5543 last_gather_stats->ip6txfrags,
5544 IRDMA_MAX_STATS_48);
5545 stats_val[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] +=
5546 IRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,
5547 last_gather_stats->ip6txmcpkts,
5548 IRDMA_MAX_STATS_48);
5549 stats_val[IRDMA_HW_STAT_INDEX_TCPRXSEGS] +=
5550 IRDMA_STATS_DELTA(gather_stats->tcprxsegs,
5551 last_gather_stats->tcprxsegs,
5552 IRDMA_MAX_STATS_48);
5553 stats_val[IRDMA_HW_STAT_INDEX_TCPTXSEG] +=
5554 IRDMA_STATS_DELTA(gather_stats->tcptxsegs,
5555 last_gather_stats->tcptxsegs,
5556 IRDMA_MAX_STATS_48);
5557 stats_val[IRDMA_HW_STAT_INDEX_RDMARXRDS] +=
5558 IRDMA_STATS_DELTA(gather_stats->rdmarxrds,
5559 last_gather_stats->rdmarxrds,
5560 IRDMA_MAX_STATS_48);
5561 stats_val[IRDMA_HW_STAT_INDEX_RDMARXSNDS] +=
5562 IRDMA_STATS_DELTA(gather_stats->rdmarxsnds,
5563 last_gather_stats->rdmarxsnds,
5564 IRDMA_MAX_STATS_48);
5565 stats_val[IRDMA_HW_STAT_INDEX_RDMARXWRS] +=
5566 IRDMA_STATS_DELTA(gather_stats->rdmarxwrs,
5567 last_gather_stats->rdmarxwrs,
5568 IRDMA_MAX_STATS_48);
5569 stats_val[IRDMA_HW_STAT_INDEX_RDMATXRDS] +=
5570 IRDMA_STATS_DELTA(gather_stats->rdmatxrds,
5571 last_gather_stats->rdmatxrds,
5572 IRDMA_MAX_STATS_48);
5573 stats_val[IRDMA_HW_STAT_INDEX_RDMATXSNDS] +=
5574 IRDMA_STATS_DELTA(gather_stats->rdmatxsnds,
5575 last_gather_stats->rdmatxsnds,
5576 IRDMA_MAX_STATS_48);
5577 stats_val[IRDMA_HW_STAT_INDEX_RDMATXWRS] +=
5578 IRDMA_STATS_DELTA(gather_stats->rdmatxwrs,
5579 last_gather_stats->rdmatxwrs,
5580 IRDMA_MAX_STATS_48);
5581 stats_val[IRDMA_HW_STAT_INDEX_RDMAVBND] +=
5582 IRDMA_STATS_DELTA(gather_stats->rdmavbn,
5583 last_gather_stats->rdmavbn,
5584 IRDMA_MAX_STATS_48);
5585 stats_val[IRDMA_HW_STAT_INDEX_RDMAVINV] +=
5586 IRDMA_STATS_DELTA(gather_stats->rdmavinv,
5587 last_gather_stats->rdmavinv,
5588 IRDMA_MAX_STATS_48);
5589 stats_val[IRDMA_HW_STAT_INDEX_UDPRXPKTS] +=
5590 IRDMA_STATS_DELTA(gather_stats->udprxpkts,
5591 last_gather_stats->udprxpkts,
5592 IRDMA_MAX_STATS_48);
5593 stats_val[IRDMA_HW_STAT_INDEX_UDPTXPKTS] +=
5594 IRDMA_STATS_DELTA(gather_stats->udptxpkts,
5595 last_gather_stats->udptxpkts,
5596 IRDMA_MAX_STATS_48);
5597 stats_val[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] +=
5598 IRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,
5599 last_gather_stats->rxnpecnmrkpkts,
5600 IRDMA_MAX_STATS_48);
5601 memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));
5602 }
5603