1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell OcteonTx2 RVU Admin Function driver
3 *
4 * Copyright (C) 2018 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12 #include <linux/pci.h>
13
14 #include "rvu_struct.h"
15 #include "rvu_reg.h"
16 #include "rvu.h"
17 #include "npc.h"
18 #include "cgx.h"
19
20 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
21 int type, int chan_id);
22
23 enum mc_tbl_sz {
24 MC_TBL_SZ_256,
25 MC_TBL_SZ_512,
26 MC_TBL_SZ_1K,
27 MC_TBL_SZ_2K,
28 MC_TBL_SZ_4K,
29 MC_TBL_SZ_8K,
30 MC_TBL_SZ_16K,
31 MC_TBL_SZ_32K,
32 MC_TBL_SZ_64K,
33 };
34
35 enum mc_buf_cnt {
36 MC_BUF_CNT_8,
37 MC_BUF_CNT_16,
38 MC_BUF_CNT_32,
39 MC_BUF_CNT_64,
40 MC_BUF_CNT_128,
41 MC_BUF_CNT_256,
42 MC_BUF_CNT_512,
43 MC_BUF_CNT_1024,
44 MC_BUF_CNT_2048,
45 };
46
47 enum nix_makr_fmt_indexes {
48 NIX_MARK_CFG_IP_DSCP_RED,
49 NIX_MARK_CFG_IP_DSCP_YELLOW,
50 NIX_MARK_CFG_IP_DSCP_YELLOW_RED,
51 NIX_MARK_CFG_IP_ECN_RED,
52 NIX_MARK_CFG_IP_ECN_YELLOW,
53 NIX_MARK_CFG_IP_ECN_YELLOW_RED,
54 NIX_MARK_CFG_VLAN_DEI_RED,
55 NIX_MARK_CFG_VLAN_DEI_YELLOW,
56 NIX_MARK_CFG_VLAN_DEI_YELLOW_RED,
57 NIX_MARK_CFG_MAX,
58 };
59
60 /* For now considering MC resources needed for broadcast
61 * pkt replication only. i.e 256 HWVFs + 12 PFs.
62 */
63 #define MC_TBL_SIZE MC_TBL_SZ_512
64 #define MC_BUF_CNT MC_BUF_CNT_128
65
66 struct mce {
67 struct hlist_node node;
68 u16 pcifunc;
69 };
70
is_nixlf_attached(struct rvu * rvu,u16 pcifunc)71 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc)
72 {
73 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
74 int blkaddr;
75
76 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
77 if (!pfvf->nixlf || blkaddr < 0)
78 return false;
79 return true;
80 }
81
rvu_get_nixlf_count(struct rvu * rvu)82 int rvu_get_nixlf_count(struct rvu *rvu)
83 {
84 struct rvu_block *block;
85 int blkaddr;
86
87 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, 0);
88 if (blkaddr < 0)
89 return 0;
90 block = &rvu->hw->block[blkaddr];
91 return block->lf.max;
92 }
93
nix_get_nixlf(struct rvu * rvu,u16 pcifunc,int * nixlf,int * nix_blkaddr)94 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr)
95 {
96 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
97 struct rvu_hwinfo *hw = rvu->hw;
98 int blkaddr;
99
100 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
101 if (!pfvf->nixlf || blkaddr < 0)
102 return NIX_AF_ERR_AF_LF_INVALID;
103
104 *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
105 if (*nixlf < 0)
106 return NIX_AF_ERR_AF_LF_INVALID;
107
108 if (nix_blkaddr)
109 *nix_blkaddr = blkaddr;
110
111 return 0;
112 }
113
nix_mce_list_init(struct nix_mce_list * list,int max)114 static void nix_mce_list_init(struct nix_mce_list *list, int max)
115 {
116 INIT_HLIST_HEAD(&list->head);
117 list->count = 0;
118 list->max = max;
119 }
120
nix_alloc_mce_list(struct nix_mcast * mcast,int count)121 static u16 nix_alloc_mce_list(struct nix_mcast *mcast, int count)
122 {
123 int idx;
124
125 if (!mcast)
126 return 0;
127
128 idx = mcast->next_free_mce;
129 mcast->next_free_mce += count;
130 return idx;
131 }
132
get_nix_hw(struct rvu_hwinfo * hw,int blkaddr)133 static inline struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr)
134 {
135 if (blkaddr == BLKADDR_NIX0 && hw->nix0)
136 return hw->nix0;
137
138 return NULL;
139 }
140
nix_rx_sync(struct rvu * rvu,int blkaddr)141 static void nix_rx_sync(struct rvu *rvu, int blkaddr)
142 {
143 int err;
144
145 /*Sync all in flight RX packets to LLC/DRAM */
146 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
147 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
148 if (err)
149 dev_err(rvu->dev, "NIX RX software sync failed\n");
150 }
151
is_valid_txschq(struct rvu * rvu,int blkaddr,int lvl,u16 pcifunc,u16 schq)152 static bool is_valid_txschq(struct rvu *rvu, int blkaddr,
153 int lvl, u16 pcifunc, u16 schq)
154 {
155 struct rvu_hwinfo *hw = rvu->hw;
156 struct nix_txsch *txsch;
157 struct nix_hw *nix_hw;
158 u16 map_func;
159
160 nix_hw = get_nix_hw(rvu->hw, blkaddr);
161 if (!nix_hw)
162 return false;
163
164 txsch = &nix_hw->txsch[lvl];
165 /* Check out of bounds */
166 if (schq >= txsch->schq.max)
167 return false;
168
169 mutex_lock(&rvu->rsrc_lock);
170 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
171 mutex_unlock(&rvu->rsrc_lock);
172
173 /* TLs aggegating traffic are shared across PF and VFs */
174 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
175 if (rvu_get_pf(map_func) != rvu_get_pf(pcifunc))
176 return false;
177 else
178 return true;
179 }
180
181 if (map_func != pcifunc)
182 return false;
183
184 return true;
185 }
186
nix_interface_init(struct rvu * rvu,u16 pcifunc,int type,int nixlf)187 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf)
188 {
189 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
190 u8 cgx_id, lmac_id;
191 int pkind, pf, vf;
192 int err;
193
194 pf = rvu_get_pf(pcifunc);
195 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
196 return 0;
197
198 switch (type) {
199 case NIX_INTF_TYPE_CGX:
200 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf];
201 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
202
203 pkind = rvu_npc_get_pkind(rvu, pf);
204 if (pkind < 0) {
205 dev_err(rvu->dev,
206 "PF_Func 0x%x: Invalid pkind\n", pcifunc);
207 return -EINVAL;
208 }
209 pfvf->rx_chan_base = NIX_CHAN_CGX_LMAC_CHX(cgx_id, lmac_id, 0);
210 pfvf->tx_chan_base = pfvf->rx_chan_base;
211 pfvf->rx_chan_cnt = 1;
212 pfvf->tx_chan_cnt = 1;
213 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind);
214 rvu_npc_set_pkind(rvu, pkind, pfvf);
215
216 /* By default we enable pause frames */
217 if ((pcifunc & RVU_PFVF_FUNC_MASK) == 0)
218 cgx_lmac_set_pause_frm(rvu_cgx_pdata(cgx_id, rvu),
219 lmac_id, true, true);
220 break;
221 case NIX_INTF_TYPE_LBK:
222 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
223
224 /* Note that AF's VFs work in pairs and talk over consecutive
225 * loopback channels.Therefore if odd number of AF VFs are
226 * enabled then the last VF remains with no pair.
227 */
228 pfvf->rx_chan_base = NIX_CHAN_LBK_CHX(0, vf);
229 pfvf->tx_chan_base = vf & 0x1 ? NIX_CHAN_LBK_CHX(0, vf - 1) :
230 NIX_CHAN_LBK_CHX(0, vf + 1);
231 pfvf->rx_chan_cnt = 1;
232 pfvf->tx_chan_cnt = 1;
233 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
234 pfvf->rx_chan_base, false);
235 break;
236 }
237
238 /* Add a UCAST forwarding rule in MCAM with this NIXLF attached
239 * RVU PF/VF's MAC address.
240 */
241 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
242 pfvf->rx_chan_base, pfvf->mac_addr);
243
244 /* Add this PF_FUNC to bcast pkt replication list */
245 err = nix_update_bcast_mce_list(rvu, pcifunc, true);
246 if (err) {
247 dev_err(rvu->dev,
248 "Bcast list, failed to enable PF_FUNC 0x%x\n",
249 pcifunc);
250 return err;
251 }
252
253 rvu_npc_install_bcast_match_entry(rvu, pcifunc,
254 nixlf, pfvf->rx_chan_base);
255 pfvf->maxlen = NIC_HW_MIN_FRS;
256 pfvf->minlen = NIC_HW_MIN_FRS;
257
258 return 0;
259 }
260
nix_interface_deinit(struct rvu * rvu,u16 pcifunc,u8 nixlf)261 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf)
262 {
263 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
264 int err;
265
266 pfvf->maxlen = 0;
267 pfvf->minlen = 0;
268 pfvf->rxvlan = false;
269
270 /* Remove this PF_FUNC from bcast pkt replication list */
271 err = nix_update_bcast_mce_list(rvu, pcifunc, false);
272 if (err) {
273 dev_err(rvu->dev,
274 "Bcast list, failed to disable PF_FUNC 0x%x\n",
275 pcifunc);
276 }
277
278 /* Free and disable any MCAM entries used by this NIX LF */
279 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
280 }
281
rvu_mbox_handler_nix_bp_disable(struct rvu * rvu,struct nix_bp_cfg_req * req,struct msg_rsp * rsp)282 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
283 struct nix_bp_cfg_req *req,
284 struct msg_rsp *rsp)
285 {
286 u16 pcifunc = req->hdr.pcifunc;
287 struct rvu_pfvf *pfvf;
288 int blkaddr, pf, type;
289 u16 chan_base, chan;
290 u64 cfg;
291
292 pf = rvu_get_pf(pcifunc);
293 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
294 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
295 return 0;
296
297 pfvf = rvu_get_pfvf(rvu, pcifunc);
298 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
299
300 chan_base = pfvf->rx_chan_base + req->chan_base;
301 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
302 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
303 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
304 cfg & ~BIT_ULL(16));
305 }
306 return 0;
307 }
308
rvu_nix_get_bpid(struct rvu * rvu,struct nix_bp_cfg_req * req,int type,int chan_id)309 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
310 int type, int chan_id)
311 {
312 int bpid, blkaddr, lmac_chan_cnt;
313 struct rvu_hwinfo *hw = rvu->hw;
314 u16 cgx_bpid_cnt, lbk_bpid_cnt;
315 struct rvu_pfvf *pfvf;
316 u8 cgx_id, lmac_id;
317 u64 cfg;
318
319 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
320 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
321 lmac_chan_cnt = cfg & 0xFF;
322
323 cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt;
324 lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF);
325
326 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
327
328 /* Backpressure IDs range division
329 * CGX channles are mapped to (0 - 191) BPIDs
330 * LBK channles are mapped to (192 - 255) BPIDs
331 * SDP channles are mapped to (256 - 511) BPIDs
332 *
333 * Lmac channles and bpids mapped as follows
334 * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15)
335 * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) ....
336 * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) ....
337 */
338 switch (type) {
339 case NIX_INTF_TYPE_CGX:
340 if ((req->chan_base + req->chan_cnt) > 15)
341 return -EINVAL;
342 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
343 /* Assign bpid based on cgx, lmac and chan id */
344 bpid = (cgx_id * hw->lmac_per_cgx * lmac_chan_cnt) +
345 (lmac_id * lmac_chan_cnt) + req->chan_base;
346
347 if (req->bpid_per_chan)
348 bpid += chan_id;
349 if (bpid > cgx_bpid_cnt)
350 return -EINVAL;
351 break;
352
353 case NIX_INTF_TYPE_LBK:
354 if ((req->chan_base + req->chan_cnt) > 63)
355 return -EINVAL;
356 bpid = cgx_bpid_cnt + req->chan_base;
357 if (req->bpid_per_chan)
358 bpid += chan_id;
359 if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt))
360 return -EINVAL;
361 break;
362 default:
363 return -EINVAL;
364 }
365 return bpid;
366 }
367
rvu_mbox_handler_nix_bp_enable(struct rvu * rvu,struct nix_bp_cfg_req * req,struct nix_bp_cfg_rsp * rsp)368 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
369 struct nix_bp_cfg_req *req,
370 struct nix_bp_cfg_rsp *rsp)
371 {
372 int blkaddr, pf, type, chan_id = 0;
373 u16 pcifunc = req->hdr.pcifunc;
374 struct rvu_pfvf *pfvf;
375 u16 chan_base, chan;
376 s16 bpid, bpid_base;
377 u64 cfg;
378
379 pf = rvu_get_pf(pcifunc);
380 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
381
382 /* Enable backpressure only for CGX mapped PFs and LBK interface */
383 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
384 return 0;
385
386 pfvf = rvu_get_pfvf(rvu, pcifunc);
387 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
388
389 bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id);
390 chan_base = pfvf->rx_chan_base + req->chan_base;
391 bpid = bpid_base;
392
393 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
394 if (bpid < 0) {
395 dev_warn(rvu->dev, "Fail to enable backpressure\n");
396 return -EINVAL;
397 }
398
399 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
400 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
401 cfg | (bpid & 0xFF) | BIT_ULL(16));
402 chan_id++;
403 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id);
404 }
405
406 for (chan = 0; chan < req->chan_cnt; chan++) {
407 /* Map channel and bpid assign to it */
408 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 |
409 (bpid_base & 0x3FF);
410 if (req->bpid_per_chan)
411 bpid_base++;
412 }
413 rsp->chan_cnt = req->chan_cnt;
414
415 return 0;
416 }
417
nix_setup_lso_tso_l3(struct rvu * rvu,int blkaddr,u64 format,bool v4,u64 * fidx)418 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr,
419 u64 format, bool v4, u64 *fidx)
420 {
421 struct nix_lso_format field = {0};
422
423 /* IP's Length field */
424 field.layer = NIX_TXLAYER_OL3;
425 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */
426 field.offset = v4 ? 2 : 4;
427 field.sizem1 = 1; /* i.e 2 bytes */
428 field.alg = NIX_LSOALG_ADD_PAYLEN;
429 rvu_write64(rvu, blkaddr,
430 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
431 *(u64 *)&field);
432
433 /* No ID field in IPv6 header */
434 if (!v4)
435 return;
436
437 /* IP's ID field */
438 field.layer = NIX_TXLAYER_OL3;
439 field.offset = 4;
440 field.sizem1 = 1; /* i.e 2 bytes */
441 field.alg = NIX_LSOALG_ADD_SEGNUM;
442 rvu_write64(rvu, blkaddr,
443 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
444 *(u64 *)&field);
445 }
446
nix_setup_lso_tso_l4(struct rvu * rvu,int blkaddr,u64 format,u64 * fidx)447 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr,
448 u64 format, u64 *fidx)
449 {
450 struct nix_lso_format field = {0};
451
452 /* TCP's sequence number field */
453 field.layer = NIX_TXLAYER_OL4;
454 field.offset = 4;
455 field.sizem1 = 3; /* i.e 4 bytes */
456 field.alg = NIX_LSOALG_ADD_OFFSET;
457 rvu_write64(rvu, blkaddr,
458 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
459 *(u64 *)&field);
460
461 /* TCP's flags field */
462 field.layer = NIX_TXLAYER_OL4;
463 field.offset = 12;
464 field.sizem1 = 1; /* 2 bytes */
465 field.alg = NIX_LSOALG_TCP_FLAGS;
466 rvu_write64(rvu, blkaddr,
467 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
468 *(u64 *)&field);
469 }
470
nix_setup_lso(struct rvu * rvu,struct nix_hw * nix_hw,int blkaddr)471 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
472 {
473 u64 cfg, idx, fidx = 0;
474
475 /* Get max HW supported format indices */
476 cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF;
477 nix_hw->lso.total = cfg;
478
479 /* Enable LSO */
480 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG);
481 /* For TSO, set first and middle segment flags to
482 * mask out PSH, RST & FIN flags in TCP packet
483 */
484 cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16));
485 cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16);
486 rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63));
487
488 /* Setup default static LSO formats
489 *
490 * Configure format fields for TCPv4 segmentation offload
491 */
492 idx = NIX_LSO_FORMAT_IDX_TSOV4;
493 nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx);
494 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
495
496 /* Set rest of the fields to NOP */
497 for (; fidx < 8; fidx++) {
498 rvu_write64(rvu, blkaddr,
499 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
500 }
501 nix_hw->lso.in_use++;
502
503 /* Configure format fields for TCPv6 segmentation offload */
504 idx = NIX_LSO_FORMAT_IDX_TSOV6;
505 fidx = 0;
506 nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx);
507 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
508
509 /* Set rest of the fields to NOP */
510 for (; fidx < 8; fidx++) {
511 rvu_write64(rvu, blkaddr,
512 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
513 }
514 nix_hw->lso.in_use++;
515 }
516
nix_ctx_free(struct rvu * rvu,struct rvu_pfvf * pfvf)517 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
518 {
519 kfree(pfvf->rq_bmap);
520 kfree(pfvf->sq_bmap);
521 kfree(pfvf->cq_bmap);
522 if (pfvf->rq_ctx)
523 qmem_free(rvu->dev, pfvf->rq_ctx);
524 if (pfvf->sq_ctx)
525 qmem_free(rvu->dev, pfvf->sq_ctx);
526 if (pfvf->cq_ctx)
527 qmem_free(rvu->dev, pfvf->cq_ctx);
528 if (pfvf->rss_ctx)
529 qmem_free(rvu->dev, pfvf->rss_ctx);
530 if (pfvf->nix_qints_ctx)
531 qmem_free(rvu->dev, pfvf->nix_qints_ctx);
532 if (pfvf->cq_ints_ctx)
533 qmem_free(rvu->dev, pfvf->cq_ints_ctx);
534
535 pfvf->rq_bmap = NULL;
536 pfvf->cq_bmap = NULL;
537 pfvf->sq_bmap = NULL;
538 pfvf->rq_ctx = NULL;
539 pfvf->sq_ctx = NULL;
540 pfvf->cq_ctx = NULL;
541 pfvf->rss_ctx = NULL;
542 pfvf->nix_qints_ctx = NULL;
543 pfvf->cq_ints_ctx = NULL;
544 }
545
nixlf_rss_ctx_init(struct rvu * rvu,int blkaddr,struct rvu_pfvf * pfvf,int nixlf,int rss_sz,int rss_grps,int hwctx_size,u64 way_mask)546 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr,
547 struct rvu_pfvf *pfvf, int nixlf,
548 int rss_sz, int rss_grps, int hwctx_size,
549 u64 way_mask)
550 {
551 int err, grp, num_indices;
552
553 /* RSS is not requested for this NIXLF */
554 if (!rss_sz)
555 return 0;
556 num_indices = rss_sz * rss_grps;
557
558 /* Alloc NIX RSS HW context memory and config the base */
559 err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size);
560 if (err)
561 return err;
562
563 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf),
564 (u64)pfvf->rss_ctx->iova);
565
566 /* Config full RSS table size, enable RSS and caching */
567 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf),
568 BIT_ULL(36) | BIT_ULL(4) |
569 ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE) |
570 way_mask << 20);
571 /* Config RSS group offset and sizes */
572 for (grp = 0; grp < rss_grps; grp++)
573 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp),
574 ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp));
575 return 0;
576 }
577
nix_aq_enqueue_wait(struct rvu * rvu,struct rvu_block * block,struct nix_aq_inst_s * inst)578 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
579 struct nix_aq_inst_s *inst)
580 {
581 struct admin_queue *aq = block->aq;
582 struct nix_aq_res_s *result;
583 int timeout = 1000;
584 u64 reg, head;
585
586 result = (struct nix_aq_res_s *)aq->res->base;
587
588 /* Get current head pointer where to append this instruction */
589 reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS);
590 head = (reg >> 4) & AQ_PTR_MASK;
591
592 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
593 (void *)inst, aq->inst->entry_sz);
594 memset(result, 0, sizeof(*result));
595 /* sync into memory */
596 wmb();
597
598 /* Ring the doorbell and wait for result */
599 rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1);
600 while (result->compcode == NIX_AQ_COMP_NOTDONE) {
601 cpu_relax();
602 udelay(1);
603 timeout--;
604 if (!timeout)
605 return -EBUSY;
606 }
607
608 if (result->compcode != NIX_AQ_COMP_GOOD)
609 /* TODO: Replace this with some error code */
610 return -EBUSY;
611
612 return 0;
613 }
614
rvu_nix_aq_enq_inst(struct rvu * rvu,struct nix_aq_enq_req * req,struct nix_aq_enq_rsp * rsp)615 static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
616 struct nix_aq_enq_rsp *rsp)
617 {
618 struct rvu_hwinfo *hw = rvu->hw;
619 u16 pcifunc = req->hdr.pcifunc;
620 int nixlf, blkaddr, rc = 0;
621 struct nix_aq_inst_s inst;
622 struct rvu_block *block;
623 struct admin_queue *aq;
624 struct rvu_pfvf *pfvf;
625 void *ctx, *mask;
626 bool ena;
627 u64 cfg;
628
629 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
630 if (blkaddr < 0)
631 return NIX_AF_ERR_AF_LF_INVALID;
632
633 block = &hw->block[blkaddr];
634 aq = block->aq;
635 if (!aq) {
636 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__);
637 return NIX_AF_ERR_AQ_ENQUEUE;
638 }
639
640 pfvf = rvu_get_pfvf(rvu, pcifunc);
641 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
642
643 /* Skip NIXLF check for broadcast MCE entry init */
644 if (!(!rsp && req->ctype == NIX_AQ_CTYPE_MCE)) {
645 if (!pfvf->nixlf || nixlf < 0)
646 return NIX_AF_ERR_AF_LF_INVALID;
647 }
648
649 switch (req->ctype) {
650 case NIX_AQ_CTYPE_RQ:
651 /* Check if index exceeds max no of queues */
652 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize)
653 rc = NIX_AF_ERR_AQ_ENQUEUE;
654 break;
655 case NIX_AQ_CTYPE_SQ:
656 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize)
657 rc = NIX_AF_ERR_AQ_ENQUEUE;
658 break;
659 case NIX_AQ_CTYPE_CQ:
660 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize)
661 rc = NIX_AF_ERR_AQ_ENQUEUE;
662 break;
663 case NIX_AQ_CTYPE_RSS:
664 /* Check if RSS is enabled and qidx is within range */
665 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf));
666 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx ||
667 (req->qidx >= (256UL << (cfg & 0xF))))
668 rc = NIX_AF_ERR_AQ_ENQUEUE;
669 break;
670 case NIX_AQ_CTYPE_MCE:
671 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG);
672 /* Check if index exceeds MCE list length */
673 if (!hw->nix0->mcast.mce_ctx ||
674 (req->qidx >= (256UL << (cfg & 0xF))))
675 rc = NIX_AF_ERR_AQ_ENQUEUE;
676
677 /* Adding multicast lists for requests from PF/VFs is not
678 * yet supported, so ignore this.
679 */
680 if (rsp)
681 rc = NIX_AF_ERR_AQ_ENQUEUE;
682 break;
683 default:
684 rc = NIX_AF_ERR_AQ_ENQUEUE;
685 }
686
687 if (rc)
688 return rc;
689
690 /* Check if SQ pointed SMQ belongs to this PF/VF or not */
691 if (req->ctype == NIX_AQ_CTYPE_SQ &&
692 ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) ||
693 (req->op == NIX_AQ_INSTOP_WRITE &&
694 req->sq_mask.ena && req->sq_mask.smq && req->sq.ena))) {
695 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ,
696 pcifunc, req->sq.smq))
697 return NIX_AF_ERR_AQ_ENQUEUE;
698 }
699
700 memset(&inst, 0, sizeof(struct nix_aq_inst_s));
701 inst.lf = nixlf;
702 inst.cindex = req->qidx;
703 inst.ctype = req->ctype;
704 inst.op = req->op;
705 /* Currently we are not supporting enqueuing multiple instructions,
706 * so always choose first entry in result memory.
707 */
708 inst.res_addr = (u64)aq->res->iova;
709
710 /* Hardware uses same aq->res->base for updating result of
711 * previous instruction hence wait here till it is done.
712 */
713 spin_lock(&aq->lock);
714
715 /* Clean result + context memory */
716 memset(aq->res->base, 0, aq->res->entry_sz);
717 /* Context needs to be written at RES_ADDR + 128 */
718 ctx = aq->res->base + 128;
719 /* Mask needs to be written at RES_ADDR + 256 */
720 mask = aq->res->base + 256;
721
722 switch (req->op) {
723 case NIX_AQ_INSTOP_WRITE:
724 if (req->ctype == NIX_AQ_CTYPE_RQ)
725 memcpy(mask, &req->rq_mask,
726 sizeof(struct nix_rq_ctx_s));
727 else if (req->ctype == NIX_AQ_CTYPE_SQ)
728 memcpy(mask, &req->sq_mask,
729 sizeof(struct nix_sq_ctx_s));
730 else if (req->ctype == NIX_AQ_CTYPE_CQ)
731 memcpy(mask, &req->cq_mask,
732 sizeof(struct nix_cq_ctx_s));
733 else if (req->ctype == NIX_AQ_CTYPE_RSS)
734 memcpy(mask, &req->rss_mask,
735 sizeof(struct nix_rsse_s));
736 else if (req->ctype == NIX_AQ_CTYPE_MCE)
737 memcpy(mask, &req->mce_mask,
738 sizeof(struct nix_rx_mce_s));
739 fallthrough;
740 case NIX_AQ_INSTOP_INIT:
741 if (req->ctype == NIX_AQ_CTYPE_RQ)
742 memcpy(ctx, &req->rq, sizeof(struct nix_rq_ctx_s));
743 else if (req->ctype == NIX_AQ_CTYPE_SQ)
744 memcpy(ctx, &req->sq, sizeof(struct nix_sq_ctx_s));
745 else if (req->ctype == NIX_AQ_CTYPE_CQ)
746 memcpy(ctx, &req->cq, sizeof(struct nix_cq_ctx_s));
747 else if (req->ctype == NIX_AQ_CTYPE_RSS)
748 memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s));
749 else if (req->ctype == NIX_AQ_CTYPE_MCE)
750 memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s));
751 break;
752 case NIX_AQ_INSTOP_NOP:
753 case NIX_AQ_INSTOP_READ:
754 case NIX_AQ_INSTOP_LOCK:
755 case NIX_AQ_INSTOP_UNLOCK:
756 break;
757 default:
758 rc = NIX_AF_ERR_AQ_ENQUEUE;
759 spin_unlock(&aq->lock);
760 return rc;
761 }
762
763 /* Submit the instruction to AQ */
764 rc = nix_aq_enqueue_wait(rvu, block, &inst);
765 if (rc) {
766 spin_unlock(&aq->lock);
767 return rc;
768 }
769
770 /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */
771 if (req->op == NIX_AQ_INSTOP_INIT) {
772 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena)
773 __set_bit(req->qidx, pfvf->rq_bmap);
774 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena)
775 __set_bit(req->qidx, pfvf->sq_bmap);
776 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena)
777 __set_bit(req->qidx, pfvf->cq_bmap);
778 }
779
780 if (req->op == NIX_AQ_INSTOP_WRITE) {
781 if (req->ctype == NIX_AQ_CTYPE_RQ) {
782 ena = (req->rq.ena & req->rq_mask.ena) |
783 (test_bit(req->qidx, pfvf->rq_bmap) &
784 ~req->rq_mask.ena);
785 if (ena)
786 __set_bit(req->qidx, pfvf->rq_bmap);
787 else
788 __clear_bit(req->qidx, pfvf->rq_bmap);
789 }
790 if (req->ctype == NIX_AQ_CTYPE_SQ) {
791 ena = (req->rq.ena & req->sq_mask.ena) |
792 (test_bit(req->qidx, pfvf->sq_bmap) &
793 ~req->sq_mask.ena);
794 if (ena)
795 __set_bit(req->qidx, pfvf->sq_bmap);
796 else
797 __clear_bit(req->qidx, pfvf->sq_bmap);
798 }
799 if (req->ctype == NIX_AQ_CTYPE_CQ) {
800 ena = (req->rq.ena & req->cq_mask.ena) |
801 (test_bit(req->qidx, pfvf->cq_bmap) &
802 ~req->cq_mask.ena);
803 if (ena)
804 __set_bit(req->qidx, pfvf->cq_bmap);
805 else
806 __clear_bit(req->qidx, pfvf->cq_bmap);
807 }
808 }
809
810 if (rsp) {
811 /* Copy read context into mailbox */
812 if (req->op == NIX_AQ_INSTOP_READ) {
813 if (req->ctype == NIX_AQ_CTYPE_RQ)
814 memcpy(&rsp->rq, ctx,
815 sizeof(struct nix_rq_ctx_s));
816 else if (req->ctype == NIX_AQ_CTYPE_SQ)
817 memcpy(&rsp->sq, ctx,
818 sizeof(struct nix_sq_ctx_s));
819 else if (req->ctype == NIX_AQ_CTYPE_CQ)
820 memcpy(&rsp->cq, ctx,
821 sizeof(struct nix_cq_ctx_s));
822 else if (req->ctype == NIX_AQ_CTYPE_RSS)
823 memcpy(&rsp->rss, ctx,
824 sizeof(struct nix_rsse_s));
825 else if (req->ctype == NIX_AQ_CTYPE_MCE)
826 memcpy(&rsp->mce, ctx,
827 sizeof(struct nix_rx_mce_s));
828 }
829 }
830
831 spin_unlock(&aq->lock);
832 return 0;
833 }
834
nix_get_ctx_name(int ctype)835 static const char *nix_get_ctx_name(int ctype)
836 {
837 switch (ctype) {
838 case NIX_AQ_CTYPE_CQ:
839 return "CQ";
840 case NIX_AQ_CTYPE_SQ:
841 return "SQ";
842 case NIX_AQ_CTYPE_RQ:
843 return "RQ";
844 case NIX_AQ_CTYPE_RSS:
845 return "RSS";
846 }
847 return "";
848 }
849
nix_lf_hwctx_disable(struct rvu * rvu,struct hwctx_disable_req * req)850 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
851 {
852 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
853 struct nix_aq_enq_req aq_req;
854 unsigned long *bmap;
855 int qidx, q_cnt = 0;
856 int err = 0, rc;
857
858 if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx)
859 return NIX_AF_ERR_AQ_ENQUEUE;
860
861 memset(&aq_req, 0, sizeof(struct nix_aq_enq_req));
862 aq_req.hdr.pcifunc = req->hdr.pcifunc;
863
864 if (req->ctype == NIX_AQ_CTYPE_CQ) {
865 aq_req.cq.ena = 0;
866 aq_req.cq_mask.ena = 1;
867 aq_req.cq.bp_ena = 0;
868 aq_req.cq_mask.bp_ena = 1;
869 q_cnt = pfvf->cq_ctx->qsize;
870 bmap = pfvf->cq_bmap;
871 }
872 if (req->ctype == NIX_AQ_CTYPE_SQ) {
873 aq_req.sq.ena = 0;
874 aq_req.sq_mask.ena = 1;
875 q_cnt = pfvf->sq_ctx->qsize;
876 bmap = pfvf->sq_bmap;
877 }
878 if (req->ctype == NIX_AQ_CTYPE_RQ) {
879 aq_req.rq.ena = 0;
880 aq_req.rq_mask.ena = 1;
881 q_cnt = pfvf->rq_ctx->qsize;
882 bmap = pfvf->rq_bmap;
883 }
884
885 aq_req.ctype = req->ctype;
886 aq_req.op = NIX_AQ_INSTOP_WRITE;
887
888 for (qidx = 0; qidx < q_cnt; qidx++) {
889 if (!test_bit(qidx, bmap))
890 continue;
891 aq_req.qidx = qidx;
892 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL);
893 if (rc) {
894 err = rc;
895 dev_err(rvu->dev, "Failed to disable %s:%d context\n",
896 nix_get_ctx_name(req->ctype), qidx);
897 }
898 }
899
900 return err;
901 }
902
903 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
nix_lf_hwctx_lockdown(struct rvu * rvu,struct nix_aq_enq_req * req)904 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req)
905 {
906 struct nix_aq_enq_req lock_ctx_req;
907 int err;
908
909 if (req->op != NIX_AQ_INSTOP_INIT)
910 return 0;
911
912 if (req->ctype == NIX_AQ_CTYPE_MCE ||
913 req->ctype == NIX_AQ_CTYPE_DYNO)
914 return 0;
915
916 memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req));
917 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
918 lock_ctx_req.ctype = req->ctype;
919 lock_ctx_req.op = NIX_AQ_INSTOP_LOCK;
920 lock_ctx_req.qidx = req->qidx;
921 err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL);
922 if (err)
923 dev_err(rvu->dev,
924 "PFUNC 0x%x: Failed to lock NIX %s:%d context\n",
925 req->hdr.pcifunc,
926 nix_get_ctx_name(req->ctype), req->qidx);
927 return err;
928 }
929
rvu_mbox_handler_nix_aq_enq(struct rvu * rvu,struct nix_aq_enq_req * req,struct nix_aq_enq_rsp * rsp)930 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
931 struct nix_aq_enq_req *req,
932 struct nix_aq_enq_rsp *rsp)
933 {
934 int err;
935
936 err = rvu_nix_aq_enq_inst(rvu, req, rsp);
937 if (!err)
938 err = nix_lf_hwctx_lockdown(rvu, req);
939 return err;
940 }
941 #else
942
rvu_mbox_handler_nix_aq_enq(struct rvu * rvu,struct nix_aq_enq_req * req,struct nix_aq_enq_rsp * rsp)943 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
944 struct nix_aq_enq_req *req,
945 struct nix_aq_enq_rsp *rsp)
946 {
947 return rvu_nix_aq_enq_inst(rvu, req, rsp);
948 }
949 #endif
950
rvu_mbox_handler_nix_hwctx_disable(struct rvu * rvu,struct hwctx_disable_req * req,struct msg_rsp * rsp)951 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu,
952 struct hwctx_disable_req *req,
953 struct msg_rsp *rsp)
954 {
955 return nix_lf_hwctx_disable(rvu, req);
956 }
957
rvu_mbox_handler_nix_lf_alloc(struct rvu * rvu,struct nix_lf_alloc_req * req,struct nix_lf_alloc_rsp * rsp)958 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu,
959 struct nix_lf_alloc_req *req,
960 struct nix_lf_alloc_rsp *rsp)
961 {
962 int nixlf, qints, hwctx_size, intf, err, rc = 0;
963 struct rvu_hwinfo *hw = rvu->hw;
964 u16 pcifunc = req->hdr.pcifunc;
965 struct rvu_block *block;
966 struct rvu_pfvf *pfvf;
967 u64 cfg, ctx_cfg;
968 int blkaddr;
969
970 if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt)
971 return NIX_AF_ERR_PARAM;
972
973 if (req->way_mask)
974 req->way_mask &= 0xFFFF;
975
976 pfvf = rvu_get_pfvf(rvu, pcifunc);
977 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
978 if (!pfvf->nixlf || blkaddr < 0)
979 return NIX_AF_ERR_AF_LF_INVALID;
980
981 block = &hw->block[blkaddr];
982 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
983 if (nixlf < 0)
984 return NIX_AF_ERR_AF_LF_INVALID;
985
986 /* Check if requested 'NIXLF <=> NPALF' mapping is valid */
987 if (req->npa_func) {
988 /* If default, use 'this' NIXLF's PFFUNC */
989 if (req->npa_func == RVU_DEFAULT_PF_FUNC)
990 req->npa_func = pcifunc;
991 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA))
992 return NIX_AF_INVAL_NPA_PF_FUNC;
993 }
994
995 /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */
996 if (req->sso_func) {
997 /* If default, use 'this' NIXLF's PFFUNC */
998 if (req->sso_func == RVU_DEFAULT_PF_FUNC)
999 req->sso_func = pcifunc;
1000 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO))
1001 return NIX_AF_INVAL_SSO_PF_FUNC;
1002 }
1003
1004 /* If RSS is being enabled, check if requested config is valid.
1005 * RSS table size should be power of two, otherwise
1006 * RSS_GRP::OFFSET + adder might go beyond that group or
1007 * won't be able to use entire table.
1008 */
1009 if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE ||
1010 !is_power_of_2(req->rss_sz)))
1011 return NIX_AF_ERR_RSS_SIZE_INVALID;
1012
1013 if (req->rss_sz &&
1014 (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS))
1015 return NIX_AF_ERR_RSS_GRPS_INVALID;
1016
1017 /* Reset this NIX LF */
1018 err = rvu_lf_reset(rvu, block, nixlf);
1019 if (err) {
1020 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1021 block->addr - BLKADDR_NIX0, nixlf);
1022 return NIX_AF_ERR_LF_RESET;
1023 }
1024
1025 ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3);
1026
1027 /* Alloc NIX RQ HW context memory and config the base */
1028 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
1029 err = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size);
1030 if (err)
1031 goto free_mem;
1032
1033 pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL);
1034 if (!pfvf->rq_bmap)
1035 goto free_mem;
1036
1037 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf),
1038 (u64)pfvf->rq_ctx->iova);
1039
1040 /* Set caching and queue count in HW */
1041 cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20;
1042 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg);
1043
1044 /* Alloc NIX SQ HW context memory and config the base */
1045 hwctx_size = 1UL << (ctx_cfg & 0xF);
1046 err = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size);
1047 if (err)
1048 goto free_mem;
1049
1050 pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL);
1051 if (!pfvf->sq_bmap)
1052 goto free_mem;
1053
1054 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf),
1055 (u64)pfvf->sq_ctx->iova);
1056
1057 cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20;
1058 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg);
1059
1060 /* Alloc NIX CQ HW context memory and config the base */
1061 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
1062 err = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size);
1063 if (err)
1064 goto free_mem;
1065
1066 pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL);
1067 if (!pfvf->cq_bmap)
1068 goto free_mem;
1069
1070 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf),
1071 (u64)pfvf->cq_ctx->iova);
1072
1073 cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20;
1074 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg);
1075
1076 /* Initialize receive side scaling (RSS) */
1077 hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF);
1078 err = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz,
1079 req->rss_grps, hwctx_size, req->way_mask);
1080 if (err)
1081 goto free_mem;
1082
1083 /* Alloc memory for CQINT's HW contexts */
1084 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1085 qints = (cfg >> 24) & 0xFFF;
1086 hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF);
1087 err = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size);
1088 if (err)
1089 goto free_mem;
1090
1091 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf),
1092 (u64)pfvf->cq_ints_ctx->iova);
1093
1094 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf),
1095 BIT_ULL(36) | req->way_mask << 20);
1096
1097 /* Alloc memory for QINT's HW contexts */
1098 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1099 qints = (cfg >> 12) & 0xFFF;
1100 hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF);
1101 err = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size);
1102 if (err)
1103 goto free_mem;
1104
1105 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf),
1106 (u64)pfvf->nix_qints_ctx->iova);
1107 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf),
1108 BIT_ULL(36) | req->way_mask << 20);
1109
1110 /* Setup VLANX TPID's.
1111 * Use VLAN1 for 802.1Q
1112 * and VLAN0 for 802.1AD.
1113 */
1114 cfg = (0x8100ULL << 16) | 0x88A8ULL;
1115 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
1116
1117 /* Enable LMTST for this NIX LF */
1118 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0));
1119
1120 /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */
1121 if (req->npa_func)
1122 cfg = req->npa_func;
1123 if (req->sso_func)
1124 cfg |= (u64)req->sso_func << 16;
1125
1126 cfg |= (u64)req->xqe_sz << 33;
1127 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg);
1128
1129 /* Config Rx pkt length, csum checks and apad enable / disable */
1130 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg);
1131
1132 /* Configure pkind for TX parse config */
1133 cfg = NPC_TX_DEF_PKIND;
1134 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg);
1135
1136 intf = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
1137 err = nix_interface_init(rvu, pcifunc, intf, nixlf);
1138 if (err)
1139 goto free_mem;
1140
1141 /* Disable NPC entries as NIXLF's contexts are not initialized yet */
1142 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
1143
1144 goto exit;
1145
1146 free_mem:
1147 nix_ctx_free(rvu, pfvf);
1148 rc = -ENOMEM;
1149
1150 exit:
1151 /* Set macaddr of this PF/VF */
1152 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
1153
1154 /* set SQB size info */
1155 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST);
1156 rsp->sqb_size = (cfg >> 34) & 0xFFFF;
1157 rsp->rx_chan_base = pfvf->rx_chan_base;
1158 rsp->tx_chan_base = pfvf->tx_chan_base;
1159 rsp->rx_chan_cnt = pfvf->rx_chan_cnt;
1160 rsp->tx_chan_cnt = pfvf->tx_chan_cnt;
1161 rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4;
1162 rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6;
1163 /* Get HW supported stat count */
1164 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
1165 rsp->lf_rx_stats = ((cfg >> 32) & 0xFF);
1166 rsp->lf_tx_stats = ((cfg >> 24) & 0xFF);
1167 /* Get count of CQ IRQs and error IRQs supported per LF */
1168 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1169 rsp->qints = ((cfg >> 12) & 0xFFF);
1170 rsp->cints = ((cfg >> 24) & 0xFFF);
1171 return rc;
1172 }
1173
rvu_mbox_handler_nix_lf_free(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)1174 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct msg_req *req,
1175 struct msg_rsp *rsp)
1176 {
1177 struct rvu_hwinfo *hw = rvu->hw;
1178 u16 pcifunc = req->hdr.pcifunc;
1179 struct rvu_block *block;
1180 int blkaddr, nixlf, err;
1181 struct rvu_pfvf *pfvf;
1182
1183 pfvf = rvu_get_pfvf(rvu, pcifunc);
1184 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1185 if (!pfvf->nixlf || blkaddr < 0)
1186 return NIX_AF_ERR_AF_LF_INVALID;
1187
1188 block = &hw->block[blkaddr];
1189 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1190 if (nixlf < 0)
1191 return NIX_AF_ERR_AF_LF_INVALID;
1192
1193 nix_interface_deinit(rvu, pcifunc, nixlf);
1194
1195 /* Reset this NIX LF */
1196 err = rvu_lf_reset(rvu, block, nixlf);
1197 if (err) {
1198 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1199 block->addr - BLKADDR_NIX0, nixlf);
1200 return NIX_AF_ERR_LF_RESET;
1201 }
1202
1203 nix_ctx_free(rvu, pfvf);
1204
1205 return 0;
1206 }
1207
rvu_mbox_handler_nix_mark_format_cfg(struct rvu * rvu,struct nix_mark_format_cfg * req,struct nix_mark_format_cfg_rsp * rsp)1208 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu,
1209 struct nix_mark_format_cfg *req,
1210 struct nix_mark_format_cfg_rsp *rsp)
1211 {
1212 u16 pcifunc = req->hdr.pcifunc;
1213 struct nix_hw *nix_hw;
1214 struct rvu_pfvf *pfvf;
1215 int blkaddr, rc;
1216 u32 cfg;
1217
1218 pfvf = rvu_get_pfvf(rvu, pcifunc);
1219 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1220 if (!pfvf->nixlf || blkaddr < 0)
1221 return NIX_AF_ERR_AF_LF_INVALID;
1222
1223 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1224 if (!nix_hw)
1225 return -EINVAL;
1226
1227 cfg = (((u32)req->offset & 0x7) << 16) |
1228 (((u32)req->y_mask & 0xF) << 12) |
1229 (((u32)req->y_val & 0xF) << 8) |
1230 (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF);
1231
1232 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg);
1233 if (rc < 0) {
1234 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)",
1235 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
1236 return NIX_AF_ERR_MARK_CFG_FAIL;
1237 }
1238
1239 rsp->mark_format_idx = rc;
1240 return 0;
1241 }
1242
1243 /* Disable shaping of pkts by a scheduler queue
1244 * at a given scheduler level.
1245 */
nix_reset_tx_shaping(struct rvu * rvu,int blkaddr,int lvl,int schq)1246 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr,
1247 int lvl, int schq)
1248 {
1249 u64 cir_reg = 0, pir_reg = 0;
1250 u64 cfg;
1251
1252 switch (lvl) {
1253 case NIX_TXSCH_LVL_TL1:
1254 cir_reg = NIX_AF_TL1X_CIR(schq);
1255 pir_reg = 0; /* PIR not available at TL1 */
1256 break;
1257 case NIX_TXSCH_LVL_TL2:
1258 cir_reg = NIX_AF_TL2X_CIR(schq);
1259 pir_reg = NIX_AF_TL2X_PIR(schq);
1260 break;
1261 case NIX_TXSCH_LVL_TL3:
1262 cir_reg = NIX_AF_TL3X_CIR(schq);
1263 pir_reg = NIX_AF_TL3X_PIR(schq);
1264 break;
1265 case NIX_TXSCH_LVL_TL4:
1266 cir_reg = NIX_AF_TL4X_CIR(schq);
1267 pir_reg = NIX_AF_TL4X_PIR(schq);
1268 break;
1269 }
1270
1271 if (!cir_reg)
1272 return;
1273 cfg = rvu_read64(rvu, blkaddr, cir_reg);
1274 rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0));
1275
1276 if (!pir_reg)
1277 return;
1278 cfg = rvu_read64(rvu, blkaddr, pir_reg);
1279 rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0));
1280 }
1281
nix_reset_tx_linkcfg(struct rvu * rvu,int blkaddr,int lvl,int schq)1282 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr,
1283 int lvl, int schq)
1284 {
1285 struct rvu_hwinfo *hw = rvu->hw;
1286 int link;
1287
1288 if (lvl >= hw->cap.nix_tx_aggr_lvl)
1289 return;
1290
1291 /* Reset TL4's SDP link config */
1292 if (lvl == NIX_TXSCH_LVL_TL4)
1293 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00);
1294
1295 if (lvl != NIX_TXSCH_LVL_TL2)
1296 return;
1297
1298 /* Reset TL2's CGX or LBK link config */
1299 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++)
1300 rvu_write64(rvu, blkaddr,
1301 NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00);
1302 }
1303
nix_get_tx_link(struct rvu * rvu,u16 pcifunc)1304 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc)
1305 {
1306 struct rvu_hwinfo *hw = rvu->hw;
1307 int pf = rvu_get_pf(pcifunc);
1308 u8 cgx_id = 0, lmac_id = 0;
1309
1310 if (is_afvf(pcifunc)) {/* LBK links */
1311 return hw->cgx_links;
1312 } else if (is_pf_cgxmapped(rvu, pf)) {
1313 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
1314 return (cgx_id * hw->lmac_per_cgx) + lmac_id;
1315 }
1316
1317 /* SDP link */
1318 return hw->cgx_links + hw->lbk_links;
1319 }
1320
nix_get_txschq_range(struct rvu * rvu,u16 pcifunc,int link,int * start,int * end)1321 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc,
1322 int link, int *start, int *end)
1323 {
1324 struct rvu_hwinfo *hw = rvu->hw;
1325 int pf = rvu_get_pf(pcifunc);
1326
1327 if (is_afvf(pcifunc)) { /* LBK links */
1328 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
1329 *end = *start + hw->cap.nix_txsch_per_lbk_lmac;
1330 } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */
1331 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
1332 *end = *start + hw->cap.nix_txsch_per_cgx_lmac;
1333 } else { /* SDP link */
1334 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) +
1335 (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links);
1336 *end = *start + hw->cap.nix_txsch_per_sdp_lmac;
1337 }
1338 }
1339
nix_check_txschq_alloc_req(struct rvu * rvu,int lvl,u16 pcifunc,struct nix_hw * nix_hw,struct nix_txsch_alloc_req * req)1340 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc,
1341 struct nix_hw *nix_hw,
1342 struct nix_txsch_alloc_req *req)
1343 {
1344 struct rvu_hwinfo *hw = rvu->hw;
1345 int schq, req_schq, free_cnt;
1346 struct nix_txsch *txsch;
1347 int link, start, end;
1348
1349 txsch = &nix_hw->txsch[lvl];
1350 req_schq = req->schq_contig[lvl] + req->schq[lvl];
1351
1352 if (!req_schq)
1353 return 0;
1354
1355 link = nix_get_tx_link(rvu, pcifunc);
1356
1357 /* For traffic aggregating scheduler level, one queue is enough */
1358 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1359 if (req_schq != 1)
1360 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1361 return 0;
1362 }
1363
1364 /* Get free SCHQ count and check if request can be accomodated */
1365 if (hw->cap.nix_fixed_txschq_mapping) {
1366 nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
1367 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK);
1368 if (end <= txsch->schq.max && schq < end &&
1369 !test_bit(schq, txsch->schq.bmap))
1370 free_cnt = 1;
1371 else
1372 free_cnt = 0;
1373 } else {
1374 free_cnt = rvu_rsrc_free_count(&txsch->schq);
1375 }
1376
1377 if (free_cnt < req_schq || req_schq > MAX_TXSCHQ_PER_FUNC)
1378 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1379
1380 /* If contiguous queues are needed, check for availability */
1381 if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] &&
1382 !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl]))
1383 return NIX_AF_ERR_TLX_ALLOC_FAIL;
1384
1385 return 0;
1386 }
1387
nix_txsch_alloc(struct rvu * rvu,struct nix_txsch * txsch,struct nix_txsch_alloc_rsp * rsp,int lvl,int start,int end)1388 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch,
1389 struct nix_txsch_alloc_rsp *rsp,
1390 int lvl, int start, int end)
1391 {
1392 struct rvu_hwinfo *hw = rvu->hw;
1393 u16 pcifunc = rsp->hdr.pcifunc;
1394 int idx, schq;
1395
1396 /* For traffic aggregating levels, queue alloc is based
1397 * on transmit link to which PF_FUNC is mapped to.
1398 */
1399 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1400 /* A single TL queue is allocated */
1401 if (rsp->schq_contig[lvl]) {
1402 rsp->schq_contig[lvl] = 1;
1403 rsp->schq_contig_list[lvl][0] = start;
1404 }
1405
1406 /* Both contig and non-contig reqs doesn't make sense here */
1407 if (rsp->schq_contig[lvl])
1408 rsp->schq[lvl] = 0;
1409
1410 if (rsp->schq[lvl]) {
1411 rsp->schq[lvl] = 1;
1412 rsp->schq_list[lvl][0] = start;
1413 }
1414 return;
1415 }
1416
1417 /* Adjust the queue request count if HW supports
1418 * only one queue per level configuration.
1419 */
1420 if (hw->cap.nix_fixed_txschq_mapping) {
1421 idx = pcifunc & RVU_PFVF_FUNC_MASK;
1422 schq = start + idx;
1423 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) {
1424 rsp->schq_contig[lvl] = 0;
1425 rsp->schq[lvl] = 0;
1426 return;
1427 }
1428
1429 if (rsp->schq_contig[lvl]) {
1430 rsp->schq_contig[lvl] = 1;
1431 set_bit(schq, txsch->schq.bmap);
1432 rsp->schq_contig_list[lvl][0] = schq;
1433 rsp->schq[lvl] = 0;
1434 } else if (rsp->schq[lvl]) {
1435 rsp->schq[lvl] = 1;
1436 set_bit(schq, txsch->schq.bmap);
1437 rsp->schq_list[lvl][0] = schq;
1438 }
1439 return;
1440 }
1441
1442 /* Allocate contiguous queue indices requesty first */
1443 if (rsp->schq_contig[lvl]) {
1444 schq = bitmap_find_next_zero_area(txsch->schq.bmap,
1445 txsch->schq.max, start,
1446 rsp->schq_contig[lvl], 0);
1447 if (schq >= end)
1448 rsp->schq_contig[lvl] = 0;
1449 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) {
1450 set_bit(schq, txsch->schq.bmap);
1451 rsp->schq_contig_list[lvl][idx] = schq;
1452 schq++;
1453 }
1454 }
1455
1456 /* Allocate non-contiguous queue indices */
1457 if (rsp->schq[lvl]) {
1458 idx = 0;
1459 for (schq = start; schq < end; schq++) {
1460 if (!test_bit(schq, txsch->schq.bmap)) {
1461 set_bit(schq, txsch->schq.bmap);
1462 rsp->schq_list[lvl][idx++] = schq;
1463 }
1464 if (idx == rsp->schq[lvl])
1465 break;
1466 }
1467 /* Update how many were allocated */
1468 rsp->schq[lvl] = idx;
1469 }
1470 }
1471
rvu_mbox_handler_nix_txsch_alloc(struct rvu * rvu,struct nix_txsch_alloc_req * req,struct nix_txsch_alloc_rsp * rsp)1472 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu,
1473 struct nix_txsch_alloc_req *req,
1474 struct nix_txsch_alloc_rsp *rsp)
1475 {
1476 struct rvu_hwinfo *hw = rvu->hw;
1477 u16 pcifunc = req->hdr.pcifunc;
1478 int link, blkaddr, rc = 0;
1479 int lvl, idx, start, end;
1480 struct nix_txsch *txsch;
1481 struct rvu_pfvf *pfvf;
1482 struct nix_hw *nix_hw;
1483 u32 *pfvf_map;
1484 u16 schq;
1485
1486 pfvf = rvu_get_pfvf(rvu, pcifunc);
1487 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1488 if (!pfvf->nixlf || blkaddr < 0)
1489 return NIX_AF_ERR_AF_LF_INVALID;
1490
1491 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1492 if (!nix_hw)
1493 return -EINVAL;
1494
1495 mutex_lock(&rvu->rsrc_lock);
1496
1497 /* Check if request is valid as per HW capabilities
1498 * and can be accomodated.
1499 */
1500 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1501 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req);
1502 if (rc)
1503 goto err;
1504 }
1505
1506 /* Allocate requested Tx scheduler queues */
1507 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1508 txsch = &nix_hw->txsch[lvl];
1509 pfvf_map = txsch->pfvf_map;
1510
1511 if (!req->schq[lvl] && !req->schq_contig[lvl])
1512 continue;
1513
1514 rsp->schq[lvl] = req->schq[lvl];
1515 rsp->schq_contig[lvl] = req->schq_contig[lvl];
1516
1517 link = nix_get_tx_link(rvu, pcifunc);
1518
1519 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
1520 start = link;
1521 end = link;
1522 } else if (hw->cap.nix_fixed_txschq_mapping) {
1523 nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
1524 } else {
1525 start = 0;
1526 end = txsch->schq.max;
1527 }
1528
1529 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end);
1530
1531 /* Reset queue config */
1532 for (idx = 0; idx < req->schq_contig[lvl]; idx++) {
1533 schq = rsp->schq_contig_list[lvl][idx];
1534 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
1535 NIX_TXSCHQ_CFG_DONE))
1536 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
1537 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
1538 nix_reset_tx_shaping(rvu, blkaddr, lvl, schq);
1539 }
1540
1541 for (idx = 0; idx < req->schq[lvl]; idx++) {
1542 schq = rsp->schq_list[lvl][idx];
1543 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
1544 NIX_TXSCHQ_CFG_DONE))
1545 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
1546 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
1547 nix_reset_tx_shaping(rvu, blkaddr, lvl, schq);
1548 }
1549 }
1550
1551 rsp->aggr_level = hw->cap.nix_tx_aggr_lvl;
1552 rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
1553 rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr,
1554 NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
1555 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
1556 goto exit;
1557 err:
1558 rc = NIX_AF_ERR_TLX_ALLOC_FAIL;
1559 exit:
1560 mutex_unlock(&rvu->rsrc_lock);
1561 return rc;
1562 }
1563
nix_smq_flush(struct rvu * rvu,int blkaddr,int smq,u16 pcifunc,int nixlf)1564 static void nix_smq_flush(struct rvu *rvu, int blkaddr,
1565 int smq, u16 pcifunc, int nixlf)
1566 {
1567 int pf = rvu_get_pf(pcifunc);
1568 u8 cgx_id = 0, lmac_id = 0;
1569 int err, restore_tx_en = 0;
1570 u64 cfg;
1571
1572 /* enable cgx tx if disabled */
1573 if (is_pf_cgxmapped(rvu, pf)) {
1574 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
1575 restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu),
1576 lmac_id, true);
1577 }
1578
1579 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
1580 /* Do SMQ flush and set enqueue xoff */
1581 cfg |= BIT_ULL(50) | BIT_ULL(49);
1582 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg);
1583
1584 /* Disable backpressure from physical link,
1585 * otherwise SMQ flush may stall.
1586 */
1587 rvu_cgx_enadis_rx_bp(rvu, pf, false);
1588
1589 /* Wait for flush to complete */
1590 err = rvu_poll_reg(rvu, blkaddr,
1591 NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true);
1592 if (err)
1593 dev_err(rvu->dev,
1594 "NIXLF%d: SMQ%d flush failed\n", nixlf, smq);
1595
1596 rvu_cgx_enadis_rx_bp(rvu, pf, true);
1597 /* restore cgx tx state */
1598 if (restore_tx_en)
1599 cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
1600 }
1601
nix_txschq_free(struct rvu * rvu,u16 pcifunc)1602 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc)
1603 {
1604 int blkaddr, nixlf, lvl, schq, err;
1605 struct rvu_hwinfo *hw = rvu->hw;
1606 struct nix_txsch *txsch;
1607 struct nix_hw *nix_hw;
1608
1609 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1610 if (blkaddr < 0)
1611 return NIX_AF_ERR_AF_LF_INVALID;
1612
1613 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1614 if (!nix_hw)
1615 return -EINVAL;
1616
1617 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
1618 if (nixlf < 0)
1619 return NIX_AF_ERR_AF_LF_INVALID;
1620
1621 /* Disable TL2/3 queue links before SMQ flush*/
1622 mutex_lock(&rvu->rsrc_lock);
1623 for (lvl = NIX_TXSCH_LVL_TL4; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1624 if (lvl != NIX_TXSCH_LVL_TL2 && lvl != NIX_TXSCH_LVL_TL4)
1625 continue;
1626
1627 txsch = &nix_hw->txsch[lvl];
1628 for (schq = 0; schq < txsch->schq.max; schq++) {
1629 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
1630 continue;
1631 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
1632 }
1633 }
1634
1635 /* Flush SMQs */
1636 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
1637 for (schq = 0; schq < txsch->schq.max; schq++) {
1638 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
1639 continue;
1640 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
1641 }
1642
1643 /* Now free scheduler queues to free pool */
1644 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1645 /* TLs above aggregation level are shared across all PF
1646 * and it's VFs, hence skip freeing them.
1647 */
1648 if (lvl >= hw->cap.nix_tx_aggr_lvl)
1649 continue;
1650
1651 txsch = &nix_hw->txsch[lvl];
1652 for (schq = 0; schq < txsch->schq.max; schq++) {
1653 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
1654 continue;
1655 rvu_free_rsrc(&txsch->schq, schq);
1656 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
1657 }
1658 }
1659 mutex_unlock(&rvu->rsrc_lock);
1660
1661 /* Sync cached info for this LF in NDC-TX to LLC/DRAM */
1662 rvu_write64(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12) | nixlf);
1663 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12), true);
1664 if (err)
1665 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf);
1666
1667 return 0;
1668 }
1669
nix_txschq_free_one(struct rvu * rvu,struct nix_txsch_free_req * req)1670 static int nix_txschq_free_one(struct rvu *rvu,
1671 struct nix_txsch_free_req *req)
1672 {
1673 struct rvu_hwinfo *hw = rvu->hw;
1674 u16 pcifunc = req->hdr.pcifunc;
1675 int lvl, schq, nixlf, blkaddr;
1676 struct nix_txsch *txsch;
1677 struct nix_hw *nix_hw;
1678 u32 *pfvf_map;
1679
1680 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1681 if (blkaddr < 0)
1682 return NIX_AF_ERR_AF_LF_INVALID;
1683
1684 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1685 if (!nix_hw)
1686 return -EINVAL;
1687
1688 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
1689 if (nixlf < 0)
1690 return NIX_AF_ERR_AF_LF_INVALID;
1691
1692 lvl = req->schq_lvl;
1693 schq = req->schq;
1694 txsch = &nix_hw->txsch[lvl];
1695
1696 if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max)
1697 return 0;
1698
1699 pfvf_map = txsch->pfvf_map;
1700 mutex_lock(&rvu->rsrc_lock);
1701
1702 if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) {
1703 mutex_unlock(&rvu->rsrc_lock);
1704 goto err;
1705 }
1706
1707 /* Flush if it is a SMQ. Onus of disabling
1708 * TL2/3 queue links before SMQ flush is on user
1709 */
1710 if (lvl == NIX_TXSCH_LVL_SMQ)
1711 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
1712
1713 /* Free the resource */
1714 rvu_free_rsrc(&txsch->schq, schq);
1715 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
1716 mutex_unlock(&rvu->rsrc_lock);
1717 return 0;
1718 err:
1719 return NIX_AF_ERR_TLX_INVALID;
1720 }
1721
rvu_mbox_handler_nix_txsch_free(struct rvu * rvu,struct nix_txsch_free_req * req,struct msg_rsp * rsp)1722 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu,
1723 struct nix_txsch_free_req *req,
1724 struct msg_rsp *rsp)
1725 {
1726 if (req->flags & TXSCHQ_FREE_ALL)
1727 return nix_txschq_free(rvu, req->hdr.pcifunc);
1728 else
1729 return nix_txschq_free_one(rvu, req);
1730 }
1731
is_txschq_hierarchy_valid(struct rvu * rvu,u16 pcifunc,int blkaddr,int lvl,u64 reg,u64 regval)1732 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr,
1733 int lvl, u64 reg, u64 regval)
1734 {
1735 u64 regbase = reg & 0xFFFF;
1736 u16 schq, parent;
1737
1738 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg))
1739 return false;
1740
1741 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
1742 /* Check if this schq belongs to this PF/VF or not */
1743 if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq))
1744 return false;
1745
1746 parent = (regval >> 16) & 0x1FF;
1747 /* Validate MDQ's TL4 parent */
1748 if (regbase == NIX_AF_MDQX_PARENT(0) &&
1749 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent))
1750 return false;
1751
1752 /* Validate TL4's TL3 parent */
1753 if (regbase == NIX_AF_TL4X_PARENT(0) &&
1754 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent))
1755 return false;
1756
1757 /* Validate TL3's TL2 parent */
1758 if (regbase == NIX_AF_TL3X_PARENT(0) &&
1759 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent))
1760 return false;
1761
1762 /* Validate TL2's TL1 parent */
1763 if (regbase == NIX_AF_TL2X_PARENT(0) &&
1764 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent))
1765 return false;
1766
1767 return true;
1768 }
1769
is_txschq_shaping_valid(struct rvu_hwinfo * hw,int lvl,u64 reg)1770 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg)
1771 {
1772 u64 regbase;
1773
1774 if (hw->cap.nix_shaping)
1775 return true;
1776
1777 /* If shaping and coloring is not supported, then
1778 * *_CIR and *_PIR registers should not be configured.
1779 */
1780 regbase = reg & 0xFFFF;
1781
1782 switch (lvl) {
1783 case NIX_TXSCH_LVL_TL1:
1784 if (regbase == NIX_AF_TL1X_CIR(0))
1785 return false;
1786 break;
1787 case NIX_TXSCH_LVL_TL2:
1788 if (regbase == NIX_AF_TL2X_CIR(0) ||
1789 regbase == NIX_AF_TL2X_PIR(0))
1790 return false;
1791 break;
1792 case NIX_TXSCH_LVL_TL3:
1793 if (regbase == NIX_AF_TL3X_CIR(0) ||
1794 regbase == NIX_AF_TL3X_PIR(0))
1795 return false;
1796 break;
1797 case NIX_TXSCH_LVL_TL4:
1798 if (regbase == NIX_AF_TL4X_CIR(0) ||
1799 regbase == NIX_AF_TL4X_PIR(0))
1800 return false;
1801 break;
1802 }
1803 return true;
1804 }
1805
nix_tl1_default_cfg(struct rvu * rvu,struct nix_hw * nix_hw,u16 pcifunc,int blkaddr)1806 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw,
1807 u16 pcifunc, int blkaddr)
1808 {
1809 u32 *pfvf_map;
1810 int schq;
1811
1812 schq = nix_get_tx_link(rvu, pcifunc);
1813 pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map;
1814 /* Skip if PF has already done the config */
1815 if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE)
1816 return;
1817 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq),
1818 (TXSCH_TL1_DFLT_RR_PRIO << 1));
1819 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
1820 TXSCH_TL1_DFLT_RR_QTM);
1821 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00);
1822 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE);
1823 }
1824
rvu_mbox_handler_nix_txschq_cfg(struct rvu * rvu,struct nix_txschq_config * req,struct msg_rsp * rsp)1825 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu,
1826 struct nix_txschq_config *req,
1827 struct msg_rsp *rsp)
1828 {
1829 struct rvu_hwinfo *hw = rvu->hw;
1830 u16 pcifunc = req->hdr.pcifunc;
1831 u64 reg, regval, schq_regbase;
1832 struct nix_txsch *txsch;
1833 struct nix_hw *nix_hw;
1834 int blkaddr, idx, err;
1835 int nixlf, schq;
1836 u32 *pfvf_map;
1837
1838 if (req->lvl >= NIX_TXSCH_LVL_CNT ||
1839 req->num_regs > MAX_REGS_PER_MBOX_MSG)
1840 return NIX_AF_INVAL_TXSCHQ_CFG;
1841
1842 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
1843 if (err)
1844 return err;
1845
1846 nix_hw = get_nix_hw(rvu->hw, blkaddr);
1847 if (!nix_hw)
1848 return -EINVAL;
1849
1850 txsch = &nix_hw->txsch[req->lvl];
1851 pfvf_map = txsch->pfvf_map;
1852
1853 if (req->lvl >= hw->cap.nix_tx_aggr_lvl &&
1854 pcifunc & RVU_PFVF_FUNC_MASK) {
1855 mutex_lock(&rvu->rsrc_lock);
1856 if (req->lvl == NIX_TXSCH_LVL_TL1)
1857 nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr);
1858 mutex_unlock(&rvu->rsrc_lock);
1859 return 0;
1860 }
1861
1862 for (idx = 0; idx < req->num_regs; idx++) {
1863 reg = req->reg[idx];
1864 regval = req->regval[idx];
1865 schq_regbase = reg & 0xFFFF;
1866
1867 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr,
1868 txsch->lvl, reg, regval))
1869 return NIX_AF_INVAL_TXSCHQ_CFG;
1870
1871 /* Check if shaping and coloring is supported */
1872 if (!is_txschq_shaping_valid(hw, req->lvl, reg))
1873 continue;
1874
1875 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */
1876 if (schq_regbase == NIX_AF_SMQX_CFG(0)) {
1877 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],
1878 pcifunc, 0);
1879 regval &= ~(0x7FULL << 24);
1880 regval |= ((u64)nixlf << 24);
1881 }
1882
1883 /* Clear 'BP_ENA' config, if it's not allowed */
1884 if (!hw->cap.nix_tx_link_bp) {
1885 if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) ||
1886 (schq_regbase & 0xFF00) ==
1887 NIX_AF_TL3_TL2X_LINKX_CFG(0, 0))
1888 regval &= ~BIT_ULL(13);
1889 }
1890
1891 /* Mark config as done for TL1 by PF */
1892 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) &&
1893 schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) {
1894 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
1895 mutex_lock(&rvu->rsrc_lock);
1896 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq],
1897 NIX_TXSCHQ_CFG_DONE);
1898 mutex_unlock(&rvu->rsrc_lock);
1899 }
1900
1901 /* SMQ flush is special hence split register writes such
1902 * that flush first and write rest of the bits later.
1903 */
1904 if (schq_regbase == NIX_AF_SMQX_CFG(0) &&
1905 (regval & BIT_ULL(49))) {
1906 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
1907 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
1908 regval &= ~BIT_ULL(49);
1909 }
1910 rvu_write64(rvu, blkaddr, reg, regval);
1911 }
1912
1913 return 0;
1914 }
1915
nix_rx_vtag_cfg(struct rvu * rvu,int nixlf,int blkaddr,struct nix_vtag_config * req)1916 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr,
1917 struct nix_vtag_config *req)
1918 {
1919 u64 regval = req->vtag_size;
1920
1921 if (req->rx.vtag_type > 7 || req->vtag_size > VTAGSIZE_T8)
1922 return -EINVAL;
1923
1924 if (req->rx.capture_vtag)
1925 regval |= BIT_ULL(5);
1926 if (req->rx.strip_vtag)
1927 regval |= BIT_ULL(4);
1928
1929 rvu_write64(rvu, blkaddr,
1930 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval);
1931 return 0;
1932 }
1933
rvu_mbox_handler_nix_vtag_cfg(struct rvu * rvu,struct nix_vtag_config * req,struct msg_rsp * rsp)1934 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu,
1935 struct nix_vtag_config *req,
1936 struct msg_rsp *rsp)
1937 {
1938 u16 pcifunc = req->hdr.pcifunc;
1939 int blkaddr, nixlf, err;
1940
1941 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
1942 if (err)
1943 return err;
1944
1945 if (req->cfg_type) {
1946 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req);
1947 if (err)
1948 return NIX_AF_ERR_PARAM;
1949 } else {
1950 /* TODO: handle tx vtag configuration */
1951 return 0;
1952 }
1953
1954 return 0;
1955 }
1956
nix_setup_mce(struct rvu * rvu,int mce,u8 op,u16 pcifunc,int next,bool eol)1957 static int nix_setup_mce(struct rvu *rvu, int mce, u8 op,
1958 u16 pcifunc, int next, bool eol)
1959 {
1960 struct nix_aq_enq_req aq_req;
1961 int err;
1962
1963 aq_req.hdr.pcifunc = 0;
1964 aq_req.ctype = NIX_AQ_CTYPE_MCE;
1965 aq_req.op = op;
1966 aq_req.qidx = mce;
1967
1968 /* Forward bcast pkts to RQ0, RSS not needed */
1969 aq_req.mce.op = 0;
1970 aq_req.mce.index = 0;
1971 aq_req.mce.eol = eol;
1972 aq_req.mce.pf_func = pcifunc;
1973 aq_req.mce.next = next;
1974
1975 /* All fields valid */
1976 *(u64 *)(&aq_req.mce_mask) = ~0ULL;
1977
1978 err = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL);
1979 if (err) {
1980 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n",
1981 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
1982 return err;
1983 }
1984 return 0;
1985 }
1986
nix_update_mce_list(struct nix_mce_list * mce_list,u16 pcifunc,bool add)1987 static int nix_update_mce_list(struct nix_mce_list *mce_list,
1988 u16 pcifunc, bool add)
1989 {
1990 struct mce *mce, *tail = NULL;
1991 bool delete = false;
1992
1993 /* Scan through the current list */
1994 hlist_for_each_entry(mce, &mce_list->head, node) {
1995 /* If already exists, then delete */
1996 if (mce->pcifunc == pcifunc && !add) {
1997 delete = true;
1998 break;
1999 }
2000 tail = mce;
2001 }
2002
2003 if (delete) {
2004 hlist_del(&mce->node);
2005 kfree(mce);
2006 mce_list->count--;
2007 return 0;
2008 }
2009
2010 if (!add)
2011 return 0;
2012
2013 /* Add a new one to the list, at the tail */
2014 mce = kzalloc(sizeof(*mce), GFP_KERNEL);
2015 if (!mce)
2016 return -ENOMEM;
2017 mce->pcifunc = pcifunc;
2018 if (!tail)
2019 hlist_add_head(&mce->node, &mce_list->head);
2020 else
2021 hlist_add_behind(&mce->node, &tail->node);
2022 mce_list->count++;
2023 return 0;
2024 }
2025
nix_update_bcast_mce_list(struct rvu * rvu,u16 pcifunc,bool add)2026 int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add)
2027 {
2028 int err = 0, idx, next_idx, last_idx;
2029 struct nix_mce_list *mce_list;
2030 struct nix_mcast *mcast;
2031 struct nix_hw *nix_hw;
2032 struct rvu_pfvf *pfvf;
2033 struct mce *mce;
2034 int blkaddr;
2035
2036 /* Broadcast pkt replication is not needed for AF's VFs, hence skip */
2037 if (is_afvf(pcifunc))
2038 return 0;
2039
2040 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2041 if (blkaddr < 0)
2042 return 0;
2043
2044 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2045 if (!nix_hw)
2046 return 0;
2047
2048 mcast = &nix_hw->mcast;
2049
2050 /* Get this PF/VF func's MCE index */
2051 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
2052 idx = pfvf->bcast_mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK);
2053
2054 mce_list = &pfvf->bcast_mce_list;
2055 if (idx > (pfvf->bcast_mce_idx + mce_list->max)) {
2056 dev_err(rvu->dev,
2057 "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n",
2058 __func__, idx, mce_list->max,
2059 pcifunc >> RVU_PFVF_PF_SHIFT);
2060 return -EINVAL;
2061 }
2062
2063 mutex_lock(&mcast->mce_lock);
2064
2065 err = nix_update_mce_list(mce_list, pcifunc, add);
2066 if (err)
2067 goto end;
2068
2069 /* Disable MCAM entry in NPC */
2070 if (!mce_list->count) {
2071 rvu_npc_enable_bcast_entry(rvu, pcifunc, false);
2072 goto end;
2073 }
2074
2075 /* Dump the updated list to HW */
2076 idx = pfvf->bcast_mce_idx;
2077 last_idx = idx + mce_list->count - 1;
2078 hlist_for_each_entry(mce, &mce_list->head, node) {
2079 if (idx > last_idx)
2080 break;
2081
2082 next_idx = idx + 1;
2083 /* EOL should be set in last MCE */
2084 err = nix_setup_mce(rvu, idx, NIX_AQ_INSTOP_WRITE,
2085 mce->pcifunc, next_idx,
2086 (next_idx > last_idx) ? true : false);
2087 if (err)
2088 goto end;
2089 idx++;
2090 }
2091
2092 end:
2093 mutex_unlock(&mcast->mce_lock);
2094 return err;
2095 }
2096
nix_setup_bcast_tables(struct rvu * rvu,struct nix_hw * nix_hw)2097 static int nix_setup_bcast_tables(struct rvu *rvu, struct nix_hw *nix_hw)
2098 {
2099 struct nix_mcast *mcast = &nix_hw->mcast;
2100 int err, pf, numvfs, idx;
2101 struct rvu_pfvf *pfvf;
2102 u16 pcifunc;
2103 u64 cfg;
2104
2105 /* Skip PF0 (i.e AF) */
2106 for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) {
2107 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2108 /* If PF is not enabled, nothing to do */
2109 if (!((cfg >> 20) & 0x01))
2110 continue;
2111 /* Get numVFs attached to this PF */
2112 numvfs = (cfg >> 12) & 0xFF;
2113
2114 pfvf = &rvu->pf[pf];
2115 /* Save the start MCE */
2116 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1);
2117
2118 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1);
2119
2120 for (idx = 0; idx < (numvfs + 1); idx++) {
2121 /* idx-0 is for PF, followed by VFs */
2122 pcifunc = (pf << RVU_PFVF_PF_SHIFT);
2123 pcifunc |= idx;
2124 /* Add dummy entries now, so that we don't have to check
2125 * for whether AQ_OP should be INIT/WRITE later on.
2126 * Will be updated when a NIXLF is attached/detached to
2127 * these PF/VFs.
2128 */
2129 err = nix_setup_mce(rvu, pfvf->bcast_mce_idx + idx,
2130 NIX_AQ_INSTOP_INIT,
2131 pcifunc, 0, true);
2132 if (err)
2133 return err;
2134 }
2135 }
2136 return 0;
2137 }
2138
nix_setup_mcast(struct rvu * rvu,struct nix_hw * nix_hw,int blkaddr)2139 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
2140 {
2141 struct nix_mcast *mcast = &nix_hw->mcast;
2142 struct rvu_hwinfo *hw = rvu->hw;
2143 int err, size;
2144
2145 size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F;
2146 size = (1ULL << size);
2147
2148 /* Alloc memory for multicast/mirror replication entries */
2149 err = qmem_alloc(rvu->dev, &mcast->mce_ctx,
2150 (256UL << MC_TBL_SIZE), size);
2151 if (err)
2152 return -ENOMEM;
2153
2154 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE,
2155 (u64)mcast->mce_ctx->iova);
2156
2157 /* Set max list length equal to max no of VFs per PF + PF itself */
2158 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG,
2159 BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE);
2160
2161 /* Alloc memory for multicast replication buffers */
2162 size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF;
2163 err = qmem_alloc(rvu->dev, &mcast->mcast_buf,
2164 (8UL << MC_BUF_CNT), size);
2165 if (err)
2166 return -ENOMEM;
2167
2168 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE,
2169 (u64)mcast->mcast_buf->iova);
2170
2171 /* Alloc pkind for NIX internal RX multicast/mirror replay */
2172 mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc);
2173
2174 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG,
2175 BIT_ULL(63) | (mcast->replay_pkind << 24) |
2176 BIT_ULL(20) | MC_BUF_CNT);
2177
2178 mutex_init(&mcast->mce_lock);
2179
2180 return nix_setup_bcast_tables(rvu, nix_hw);
2181 }
2182
nix_setup_txschq(struct rvu * rvu,struct nix_hw * nix_hw,int blkaddr)2183 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
2184 {
2185 struct nix_txsch *txsch;
2186 int err, lvl, schq;
2187 u64 cfg, reg;
2188
2189 /* Get scheduler queue count of each type and alloc
2190 * bitmap for each for alloc/free/attach operations.
2191 */
2192 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2193 txsch = &nix_hw->txsch[lvl];
2194 txsch->lvl = lvl;
2195 switch (lvl) {
2196 case NIX_TXSCH_LVL_SMQ:
2197 reg = NIX_AF_MDQ_CONST;
2198 break;
2199 case NIX_TXSCH_LVL_TL4:
2200 reg = NIX_AF_TL4_CONST;
2201 break;
2202 case NIX_TXSCH_LVL_TL3:
2203 reg = NIX_AF_TL3_CONST;
2204 break;
2205 case NIX_TXSCH_LVL_TL2:
2206 reg = NIX_AF_TL2_CONST;
2207 break;
2208 case NIX_TXSCH_LVL_TL1:
2209 reg = NIX_AF_TL1_CONST;
2210 break;
2211 }
2212 cfg = rvu_read64(rvu, blkaddr, reg);
2213 txsch->schq.max = cfg & 0xFFFF;
2214 err = rvu_alloc_bitmap(&txsch->schq);
2215 if (err)
2216 return err;
2217
2218 /* Allocate memory for scheduler queues to
2219 * PF/VF pcifunc mapping info.
2220 */
2221 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max,
2222 sizeof(u32), GFP_KERNEL);
2223 if (!txsch->pfvf_map)
2224 return -ENOMEM;
2225 for (schq = 0; schq < txsch->schq.max; schq++)
2226 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
2227 }
2228 return 0;
2229 }
2230
rvu_nix_reserve_mark_format(struct rvu * rvu,struct nix_hw * nix_hw,int blkaddr,u32 cfg)2231 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw,
2232 int blkaddr, u32 cfg)
2233 {
2234 int fmt_idx;
2235
2236 for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) {
2237 if (nix_hw->mark_format.cfg[fmt_idx] == cfg)
2238 return fmt_idx;
2239 }
2240 if (fmt_idx >= nix_hw->mark_format.total)
2241 return -ERANGE;
2242
2243 rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg);
2244 nix_hw->mark_format.cfg[fmt_idx] = cfg;
2245 nix_hw->mark_format.in_use++;
2246 return fmt_idx;
2247 }
2248
nix_af_mark_format_setup(struct rvu * rvu,struct nix_hw * nix_hw,int blkaddr)2249 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw,
2250 int blkaddr)
2251 {
2252 u64 cfgs[] = {
2253 [NIX_MARK_CFG_IP_DSCP_RED] = 0x10003,
2254 [NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200,
2255 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203,
2256 [NIX_MARK_CFG_IP_ECN_RED] = 0x6000c,
2257 [NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00,
2258 [NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c,
2259 [NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008,
2260 [NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800,
2261 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808,
2262 };
2263 int i, rc;
2264 u64 total;
2265
2266 total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8;
2267 nix_hw->mark_format.total = (u8)total;
2268 nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32),
2269 GFP_KERNEL);
2270 if (!nix_hw->mark_format.cfg)
2271 return -ENOMEM;
2272 for (i = 0; i < NIX_MARK_CFG_MAX; i++) {
2273 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]);
2274 if (rc < 0)
2275 dev_err(rvu->dev, "Err %d in setup mark format %d\n",
2276 i, rc);
2277 }
2278
2279 return 0;
2280 }
2281
rvu_mbox_handler_nix_stats_rst(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)2282 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req,
2283 struct msg_rsp *rsp)
2284 {
2285 u16 pcifunc = req->hdr.pcifunc;
2286 int i, nixlf, blkaddr, err;
2287 u64 stats;
2288
2289 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2290 if (err)
2291 return err;
2292
2293 /* Get stats count supported by HW */
2294 stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
2295
2296 /* Reset tx stats */
2297 for (i = 0; i < ((stats >> 24) & 0xFF); i++)
2298 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0);
2299
2300 /* Reset rx stats */
2301 for (i = 0; i < ((stats >> 32) & 0xFF); i++)
2302 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0);
2303
2304 return 0;
2305 }
2306
2307 /* Returns the ALG index to be set into NPC_RX_ACTION */
get_flowkey_alg_idx(struct nix_hw * nix_hw,u32 flow_cfg)2308 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg)
2309 {
2310 int i;
2311
2312 /* Scan over exiting algo entries to find a match */
2313 for (i = 0; i < nix_hw->flowkey.in_use; i++)
2314 if (nix_hw->flowkey.flowkey[i] == flow_cfg)
2315 return i;
2316
2317 return -ERANGE;
2318 }
2319
set_flowkey_fields(struct nix_rx_flowkey_alg * alg,u32 flow_cfg)2320 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg)
2321 {
2322 int idx, nr_field, key_off, field_marker, keyoff_marker;
2323 int max_key_off, max_bit_pos, group_member;
2324 struct nix_rx_flowkey_alg *field;
2325 struct nix_rx_flowkey_alg tmp;
2326 u32 key_type, valid_key;
2327
2328 if (!alg)
2329 return -EINVAL;
2330
2331 #define FIELDS_PER_ALG 5
2332 #define MAX_KEY_OFF 40
2333 /* Clear all fields */
2334 memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG);
2335
2336 /* Each of the 32 possible flow key algorithm definitions should
2337 * fall into above incremental config (except ALG0). Otherwise a
2338 * single NPC MCAM entry is not sufficient for supporting RSS.
2339 *
2340 * If a different definition or combination needed then NPC MCAM
2341 * has to be programmed to filter such pkts and it's action should
2342 * point to this definition to calculate flowtag or hash.
2343 *
2344 * The `for loop` goes over _all_ protocol field and the following
2345 * variables depicts the state machine forward progress logic.
2346 *
2347 * keyoff_marker - Enabled when hash byte length needs to be accounted
2348 * in field->key_offset update.
2349 * field_marker - Enabled when a new field needs to be selected.
2350 * group_member - Enabled when protocol is part of a group.
2351 */
2352
2353 keyoff_marker = 0; max_key_off = 0; group_member = 0;
2354 nr_field = 0; key_off = 0; field_marker = 1;
2355 field = &tmp; max_bit_pos = fls(flow_cfg);
2356 for (idx = 0;
2357 idx < max_bit_pos && nr_field < FIELDS_PER_ALG &&
2358 key_off < MAX_KEY_OFF; idx++) {
2359 key_type = BIT(idx);
2360 valid_key = flow_cfg & key_type;
2361 /* Found a field marker, reset the field values */
2362 if (field_marker)
2363 memset(&tmp, 0, sizeof(tmp));
2364
2365 field_marker = true;
2366 keyoff_marker = true;
2367 switch (key_type) {
2368 case NIX_FLOW_KEY_TYPE_PORT:
2369 field->sel_chan = true;
2370 /* This should be set to 1, when SEL_CHAN is set */
2371 field->bytesm1 = 1;
2372 break;
2373 case NIX_FLOW_KEY_TYPE_IPV4:
2374 case NIX_FLOW_KEY_TYPE_INNR_IPV4:
2375 field->lid = NPC_LID_LC;
2376 field->ltype_match = NPC_LT_LC_IP;
2377 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) {
2378 field->lid = NPC_LID_LG;
2379 field->ltype_match = NPC_LT_LG_TU_IP;
2380 }
2381 field->hdr_offset = 12; /* SIP offset */
2382 field->bytesm1 = 7; /* SIP + DIP, 8 bytes */
2383 field->ltype_mask = 0xF; /* Match only IPv4 */
2384 keyoff_marker = false;
2385 break;
2386 case NIX_FLOW_KEY_TYPE_IPV6:
2387 case NIX_FLOW_KEY_TYPE_INNR_IPV6:
2388 field->lid = NPC_LID_LC;
2389 field->ltype_match = NPC_LT_LC_IP6;
2390 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) {
2391 field->lid = NPC_LID_LG;
2392 field->ltype_match = NPC_LT_LG_TU_IP6;
2393 }
2394 field->hdr_offset = 8; /* SIP offset */
2395 field->bytesm1 = 31; /* SIP + DIP, 32 bytes */
2396 field->ltype_mask = 0xF; /* Match only IPv6 */
2397 break;
2398 case NIX_FLOW_KEY_TYPE_TCP:
2399 case NIX_FLOW_KEY_TYPE_UDP:
2400 case NIX_FLOW_KEY_TYPE_SCTP:
2401 case NIX_FLOW_KEY_TYPE_INNR_TCP:
2402 case NIX_FLOW_KEY_TYPE_INNR_UDP:
2403 case NIX_FLOW_KEY_TYPE_INNR_SCTP:
2404 field->lid = NPC_LID_LD;
2405 if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP ||
2406 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP ||
2407 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP)
2408 field->lid = NPC_LID_LH;
2409 field->bytesm1 = 3; /* Sport + Dport, 4 bytes */
2410
2411 /* Enum values for NPC_LID_LD and NPC_LID_LG are same,
2412 * so no need to change the ltype_match, just change
2413 * the lid for inner protocols
2414 */
2415 BUILD_BUG_ON((int)NPC_LT_LD_TCP !=
2416 (int)NPC_LT_LH_TU_TCP);
2417 BUILD_BUG_ON((int)NPC_LT_LD_UDP !=
2418 (int)NPC_LT_LH_TU_UDP);
2419 BUILD_BUG_ON((int)NPC_LT_LD_SCTP !=
2420 (int)NPC_LT_LH_TU_SCTP);
2421
2422 if ((key_type == NIX_FLOW_KEY_TYPE_TCP ||
2423 key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) &&
2424 valid_key) {
2425 field->ltype_match |= NPC_LT_LD_TCP;
2426 group_member = true;
2427 } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP ||
2428 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) &&
2429 valid_key) {
2430 field->ltype_match |= NPC_LT_LD_UDP;
2431 group_member = true;
2432 } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP ||
2433 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) &&
2434 valid_key) {
2435 field->ltype_match |= NPC_LT_LD_SCTP;
2436 group_member = true;
2437 }
2438 field->ltype_mask = ~field->ltype_match;
2439 if (key_type == NIX_FLOW_KEY_TYPE_SCTP ||
2440 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) {
2441 /* Handle the case where any of the group item
2442 * is enabled in the group but not the final one
2443 */
2444 if (group_member) {
2445 valid_key = true;
2446 group_member = false;
2447 }
2448 } else {
2449 field_marker = false;
2450 keyoff_marker = false;
2451 }
2452 break;
2453 case NIX_FLOW_KEY_TYPE_NVGRE:
2454 field->lid = NPC_LID_LD;
2455 field->hdr_offset = 4; /* VSID offset */
2456 field->bytesm1 = 2;
2457 field->ltype_match = NPC_LT_LD_NVGRE;
2458 field->ltype_mask = 0xF;
2459 break;
2460 case NIX_FLOW_KEY_TYPE_VXLAN:
2461 case NIX_FLOW_KEY_TYPE_GENEVE:
2462 field->lid = NPC_LID_LE;
2463 field->bytesm1 = 2;
2464 field->hdr_offset = 4;
2465 field->ltype_mask = 0xF;
2466 field_marker = false;
2467 keyoff_marker = false;
2468
2469 if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) {
2470 field->ltype_match |= NPC_LT_LE_VXLAN;
2471 group_member = true;
2472 }
2473
2474 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) {
2475 field->ltype_match |= NPC_LT_LE_GENEVE;
2476 group_member = true;
2477 }
2478
2479 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) {
2480 if (group_member) {
2481 field->ltype_mask = ~field->ltype_match;
2482 field_marker = true;
2483 keyoff_marker = true;
2484 valid_key = true;
2485 group_member = false;
2486 }
2487 }
2488 break;
2489 case NIX_FLOW_KEY_TYPE_ETH_DMAC:
2490 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC:
2491 field->lid = NPC_LID_LA;
2492 field->ltype_match = NPC_LT_LA_ETHER;
2493 if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) {
2494 field->lid = NPC_LID_LF;
2495 field->ltype_match = NPC_LT_LF_TU_ETHER;
2496 }
2497 field->hdr_offset = 0;
2498 field->bytesm1 = 5; /* DMAC 6 Byte */
2499 field->ltype_mask = 0xF;
2500 break;
2501 case NIX_FLOW_KEY_TYPE_IPV6_EXT:
2502 field->lid = NPC_LID_LC;
2503 field->hdr_offset = 40; /* IPV6 hdr */
2504 field->bytesm1 = 0; /* 1 Byte ext hdr*/
2505 field->ltype_match = NPC_LT_LC_IP6_EXT;
2506 field->ltype_mask = 0xF;
2507 break;
2508 case NIX_FLOW_KEY_TYPE_GTPU:
2509 field->lid = NPC_LID_LE;
2510 field->hdr_offset = 4;
2511 field->bytesm1 = 3; /* 4 bytes TID*/
2512 field->ltype_match = NPC_LT_LE_GTPU;
2513 field->ltype_mask = 0xF;
2514 break;
2515 case NIX_FLOW_KEY_TYPE_VLAN:
2516 field->lid = NPC_LID_LB;
2517 field->hdr_offset = 2; /* Skip TPID (2-bytes) */
2518 field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */
2519 field->ltype_match = NPC_LT_LB_CTAG;
2520 field->ltype_mask = 0xF;
2521 field->fn_mask = 1; /* Mask out the first nibble */
2522 break;
2523 }
2524 field->ena = 1;
2525
2526 /* Found a valid flow key type */
2527 if (valid_key) {
2528 field->key_offset = key_off;
2529 memcpy(&alg[nr_field], field, sizeof(*field));
2530 max_key_off = max(max_key_off, field->bytesm1 + 1);
2531
2532 /* Found a field marker, get the next field */
2533 if (field_marker)
2534 nr_field++;
2535 }
2536
2537 /* Found a keyoff marker, update the new key_off */
2538 if (keyoff_marker) {
2539 key_off += max_key_off;
2540 max_key_off = 0;
2541 }
2542 }
2543 /* Processed all the flow key types */
2544 if (idx == max_bit_pos && key_off <= MAX_KEY_OFF)
2545 return 0;
2546 else
2547 return NIX_AF_ERR_RSS_NOSPC_FIELD;
2548 }
2549
reserve_flowkey_alg_idx(struct rvu * rvu,int blkaddr,u32 flow_cfg)2550 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg)
2551 {
2552 u64 field[FIELDS_PER_ALG];
2553 struct nix_hw *hw;
2554 int fid, rc;
2555
2556 hw = get_nix_hw(rvu->hw, blkaddr);
2557 if (!hw)
2558 return -EINVAL;
2559
2560 /* No room to add new flow hash algoritham */
2561 if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX)
2562 return NIX_AF_ERR_RSS_NOSPC_ALGO;
2563
2564 /* Generate algo fields for the given flow_cfg */
2565 rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg);
2566 if (rc)
2567 return rc;
2568
2569 /* Update ALGX_FIELDX register with generated fields */
2570 for (fid = 0; fid < FIELDS_PER_ALG; fid++)
2571 rvu_write64(rvu, blkaddr,
2572 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use,
2573 fid), field[fid]);
2574
2575 /* Store the flow_cfg for futher lookup */
2576 rc = hw->flowkey.in_use;
2577 hw->flowkey.flowkey[rc] = flow_cfg;
2578 hw->flowkey.in_use++;
2579
2580 return rc;
2581 }
2582
rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu * rvu,struct nix_rss_flowkey_cfg * req,struct nix_rss_flowkey_cfg_rsp * rsp)2583 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu,
2584 struct nix_rss_flowkey_cfg *req,
2585 struct nix_rss_flowkey_cfg_rsp *rsp)
2586 {
2587 u16 pcifunc = req->hdr.pcifunc;
2588 int alg_idx, nixlf, blkaddr;
2589 struct nix_hw *nix_hw;
2590 int err;
2591
2592 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2593 if (err)
2594 return err;
2595
2596 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2597 if (!nix_hw)
2598 return -EINVAL;
2599
2600 alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg);
2601 /* Failed to get algo index from the exiting list, reserve new */
2602 if (alg_idx < 0) {
2603 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr,
2604 req->flowkey_cfg);
2605 if (alg_idx < 0)
2606 return alg_idx;
2607 }
2608 rsp->alg_idx = alg_idx;
2609 rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group,
2610 alg_idx, req->mcam_index);
2611 return 0;
2612 }
2613
nix_rx_flowkey_alg_cfg(struct rvu * rvu,int blkaddr)2614 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr)
2615 {
2616 u32 flowkey_cfg, minkey_cfg;
2617 int alg, fid, rc;
2618
2619 /* Disable all flow key algx fieldx */
2620 for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) {
2621 for (fid = 0; fid < FIELDS_PER_ALG; fid++)
2622 rvu_write64(rvu, blkaddr,
2623 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid),
2624 0);
2625 }
2626
2627 /* IPv4/IPv6 SIP/DIPs */
2628 flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6;
2629 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2630 if (rc < 0)
2631 return rc;
2632
2633 /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
2634 minkey_cfg = flowkey_cfg;
2635 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP;
2636 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2637 if (rc < 0)
2638 return rc;
2639
2640 /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
2641 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP;
2642 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2643 if (rc < 0)
2644 return rc;
2645
2646 /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
2647 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP;
2648 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2649 if (rc < 0)
2650 return rc;
2651
2652 /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */
2653 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
2654 NIX_FLOW_KEY_TYPE_UDP;
2655 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2656 if (rc < 0)
2657 return rc;
2658
2659 /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
2660 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
2661 NIX_FLOW_KEY_TYPE_SCTP;
2662 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2663 if (rc < 0)
2664 return rc;
2665
2666 /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
2667 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP |
2668 NIX_FLOW_KEY_TYPE_SCTP;
2669 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2670 if (rc < 0)
2671 return rc;
2672
2673 /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
2674 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
2675 NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP;
2676 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
2677 if (rc < 0)
2678 return rc;
2679
2680 return 0;
2681 }
2682
rvu_mbox_handler_nix_set_mac_addr(struct rvu * rvu,struct nix_set_mac_addr * req,struct msg_rsp * rsp)2683 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu,
2684 struct nix_set_mac_addr *req,
2685 struct msg_rsp *rsp)
2686 {
2687 u16 pcifunc = req->hdr.pcifunc;
2688 int blkaddr, nixlf, err;
2689 struct rvu_pfvf *pfvf;
2690
2691 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2692 if (err)
2693 return err;
2694
2695 pfvf = rvu_get_pfvf(rvu, pcifunc);
2696
2697 ether_addr_copy(pfvf->mac_addr, req->mac_addr);
2698
2699 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
2700 pfvf->rx_chan_base, req->mac_addr);
2701
2702 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf);
2703
2704 return 0;
2705 }
2706
rvu_mbox_handler_nix_get_mac_addr(struct rvu * rvu,struct msg_req * req,struct nix_get_mac_addr_rsp * rsp)2707 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu,
2708 struct msg_req *req,
2709 struct nix_get_mac_addr_rsp *rsp)
2710 {
2711 u16 pcifunc = req->hdr.pcifunc;
2712 struct rvu_pfvf *pfvf;
2713
2714 if (!is_nixlf_attached(rvu, pcifunc))
2715 return NIX_AF_ERR_AF_LF_INVALID;
2716
2717 pfvf = rvu_get_pfvf(rvu, pcifunc);
2718
2719 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
2720
2721 return 0;
2722 }
2723
rvu_mbox_handler_nix_set_rx_mode(struct rvu * rvu,struct nix_rx_mode * req,struct msg_rsp * rsp)2724 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req,
2725 struct msg_rsp *rsp)
2726 {
2727 bool allmulti = false, disable_promisc = false;
2728 u16 pcifunc = req->hdr.pcifunc;
2729 int blkaddr, nixlf, err;
2730 struct rvu_pfvf *pfvf;
2731
2732 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2733 if (err)
2734 return err;
2735
2736 pfvf = rvu_get_pfvf(rvu, pcifunc);
2737
2738 if (req->mode & NIX_RX_MODE_PROMISC)
2739 allmulti = false;
2740 else if (req->mode & NIX_RX_MODE_ALLMULTI)
2741 allmulti = true;
2742 else
2743 disable_promisc = true;
2744
2745 if (disable_promisc)
2746 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf);
2747 else
2748 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
2749 pfvf->rx_chan_base, allmulti);
2750
2751 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf);
2752
2753 return 0;
2754 }
2755
nix_find_link_frs(struct rvu * rvu,struct nix_frs_cfg * req,u16 pcifunc)2756 static void nix_find_link_frs(struct rvu *rvu,
2757 struct nix_frs_cfg *req, u16 pcifunc)
2758 {
2759 int pf = rvu_get_pf(pcifunc);
2760 struct rvu_pfvf *pfvf;
2761 int maxlen, minlen;
2762 int numvfs, hwvf;
2763 int vf;
2764
2765 /* Update with requester's min/max lengths */
2766 pfvf = rvu_get_pfvf(rvu, pcifunc);
2767 pfvf->maxlen = req->maxlen;
2768 if (req->update_minlen)
2769 pfvf->minlen = req->minlen;
2770
2771 maxlen = req->maxlen;
2772 minlen = req->update_minlen ? req->minlen : 0;
2773
2774 /* Get this PF's numVFs and starting hwvf */
2775 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
2776
2777 /* For each VF, compare requested max/minlen */
2778 for (vf = 0; vf < numvfs; vf++) {
2779 pfvf = &rvu->hwvf[hwvf + vf];
2780 if (pfvf->maxlen > maxlen)
2781 maxlen = pfvf->maxlen;
2782 if (req->update_minlen &&
2783 pfvf->minlen && pfvf->minlen < minlen)
2784 minlen = pfvf->minlen;
2785 }
2786
2787 /* Compare requested max/minlen with PF's max/minlen */
2788 pfvf = &rvu->pf[pf];
2789 if (pfvf->maxlen > maxlen)
2790 maxlen = pfvf->maxlen;
2791 if (req->update_minlen &&
2792 pfvf->minlen && pfvf->minlen < minlen)
2793 minlen = pfvf->minlen;
2794
2795 /* Update the request with max/min PF's and it's VF's max/min */
2796 req->maxlen = maxlen;
2797 if (req->update_minlen)
2798 req->minlen = minlen;
2799 }
2800
rvu_mbox_handler_nix_set_hw_frs(struct rvu * rvu,struct nix_frs_cfg * req,struct msg_rsp * rsp)2801 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req,
2802 struct msg_rsp *rsp)
2803 {
2804 struct rvu_hwinfo *hw = rvu->hw;
2805 u16 pcifunc = req->hdr.pcifunc;
2806 int pf = rvu_get_pf(pcifunc);
2807 int blkaddr, schq, link = -1;
2808 struct nix_txsch *txsch;
2809 u64 cfg, lmac_fifo_len;
2810 struct nix_hw *nix_hw;
2811 u8 cgx = 0, lmac = 0;
2812
2813 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2814 if (blkaddr < 0)
2815 return NIX_AF_ERR_AF_LF_INVALID;
2816
2817 nix_hw = get_nix_hw(rvu->hw, blkaddr);
2818 if (!nix_hw)
2819 return -EINVAL;
2820
2821 if (!req->sdp_link && req->maxlen > NIC_HW_MAX_FRS)
2822 return NIX_AF_ERR_FRS_INVALID;
2823
2824 if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS)
2825 return NIX_AF_ERR_FRS_INVALID;
2826
2827 /* Check if requester wants to update SMQ's */
2828 if (!req->update_smq)
2829 goto rx_frscfg;
2830
2831 /* Update min/maxlen in each of the SMQ attached to this PF/VF */
2832 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
2833 mutex_lock(&rvu->rsrc_lock);
2834 for (schq = 0; schq < txsch->schq.max; schq++) {
2835 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2836 continue;
2837 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq));
2838 cfg = (cfg & ~(0xFFFFULL << 8)) | ((u64)req->maxlen << 8);
2839 if (req->update_minlen)
2840 cfg = (cfg & ~0x7FULL) | ((u64)req->minlen & 0x7F);
2841 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq), cfg);
2842 }
2843 mutex_unlock(&rvu->rsrc_lock);
2844
2845 rx_frscfg:
2846 /* Check if config is for SDP link */
2847 if (req->sdp_link) {
2848 if (!hw->sdp_links)
2849 return NIX_AF_ERR_RX_LINK_INVALID;
2850 link = hw->cgx_links + hw->lbk_links;
2851 goto linkcfg;
2852 }
2853
2854 /* Check if the request is from CGX mapped RVU PF */
2855 if (is_pf_cgxmapped(rvu, pf)) {
2856 /* Get CGX and LMAC to which this PF is mapped and find link */
2857 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac);
2858 link = (cgx * hw->lmac_per_cgx) + lmac;
2859 } else if (pf == 0) {
2860 /* For VFs of PF0 ingress is LBK port, so config LBK link */
2861 link = hw->cgx_links;
2862 }
2863
2864 if (link < 0)
2865 return NIX_AF_ERR_RX_LINK_INVALID;
2866
2867 nix_find_link_frs(rvu, req, pcifunc);
2868
2869 linkcfg:
2870 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link));
2871 cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16);
2872 if (req->update_minlen)
2873 cfg = (cfg & ~0xFFFFULL) | req->minlen;
2874 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg);
2875
2876 if (req->sdp_link || pf == 0)
2877 return 0;
2878
2879 /* Update transmit credits for CGX links */
2880 lmac_fifo_len =
2881 CGX_FIFO_LEN / cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
2882 cfg = rvu_read64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link));
2883 cfg &= ~(0xFFFFFULL << 12);
2884 cfg |= ((lmac_fifo_len - req->maxlen) / 16) << 12;
2885 rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg);
2886 return 0;
2887 }
2888
rvu_mbox_handler_nix_rxvlan_alloc(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)2889 int rvu_mbox_handler_nix_rxvlan_alloc(struct rvu *rvu, struct msg_req *req,
2890 struct msg_rsp *rsp)
2891 {
2892 struct npc_mcam_alloc_entry_req alloc_req = { };
2893 struct npc_mcam_alloc_entry_rsp alloc_rsp = { };
2894 struct npc_mcam_free_entry_req free_req = { };
2895 u16 pcifunc = req->hdr.pcifunc;
2896 int blkaddr, nixlf, err;
2897 struct rvu_pfvf *pfvf;
2898
2899 /* LBK VFs do not have separate MCAM UCAST entry hence
2900 * skip allocating rxvlan for them
2901 */
2902 if (is_afvf(pcifunc))
2903 return 0;
2904
2905 pfvf = rvu_get_pfvf(rvu, pcifunc);
2906 if (pfvf->rxvlan)
2907 return 0;
2908
2909 /* alloc new mcam entry */
2910 alloc_req.hdr.pcifunc = pcifunc;
2911 alloc_req.count = 1;
2912
2913 err = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &alloc_req,
2914 &alloc_rsp);
2915 if (err)
2916 return err;
2917
2918 /* update entry to enable rxvlan offload */
2919 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2920 if (blkaddr < 0) {
2921 err = NIX_AF_ERR_AF_LF_INVALID;
2922 goto free_entry;
2923 }
2924
2925 nixlf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr], pcifunc, 0);
2926 if (nixlf < 0) {
2927 err = NIX_AF_ERR_AF_LF_INVALID;
2928 goto free_entry;
2929 }
2930
2931 pfvf->rxvlan_index = alloc_rsp.entry_list[0];
2932 /* all it means is that rxvlan_index is valid */
2933 pfvf->rxvlan = true;
2934
2935 err = rvu_npc_update_rxvlan(rvu, pcifunc, nixlf);
2936 if (err)
2937 goto free_entry;
2938
2939 return 0;
2940 free_entry:
2941 free_req.hdr.pcifunc = pcifunc;
2942 free_req.entry = alloc_rsp.entry_list[0];
2943 rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, rsp);
2944 pfvf->rxvlan = false;
2945 return err;
2946 }
2947
rvu_mbox_handler_nix_set_rx_cfg(struct rvu * rvu,struct nix_rx_cfg * req,struct msg_rsp * rsp)2948 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req,
2949 struct msg_rsp *rsp)
2950 {
2951 int nixlf, blkaddr, err;
2952 u64 cfg;
2953
2954 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr);
2955 if (err)
2956 return err;
2957
2958 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf));
2959 /* Set the interface configuration */
2960 if (req->len_verify & BIT(0))
2961 cfg |= BIT_ULL(41);
2962 else
2963 cfg &= ~BIT_ULL(41);
2964
2965 if (req->len_verify & BIT(1))
2966 cfg |= BIT_ULL(40);
2967 else
2968 cfg &= ~BIT_ULL(40);
2969
2970 if (req->csum_verify & BIT(0))
2971 cfg |= BIT_ULL(37);
2972 else
2973 cfg &= ~BIT_ULL(37);
2974
2975 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg);
2976
2977 return 0;
2978 }
2979
nix_link_config(struct rvu * rvu,int blkaddr)2980 static void nix_link_config(struct rvu *rvu, int blkaddr)
2981 {
2982 struct rvu_hwinfo *hw = rvu->hw;
2983 int cgx, lmac_cnt, slink, link;
2984 u64 tx_credits;
2985
2986 /* Set default min/max packet lengths allowed on NIX Rx links.
2987 *
2988 * With HW reset minlen value of 60byte, HW will treat ARP pkts
2989 * as undersize and report them to SW as error pkts, hence
2990 * setting it to 40 bytes.
2991 */
2992 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++) {
2993 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
2994 NIC_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
2995 }
2996
2997 if (hw->sdp_links) {
2998 link = hw->cgx_links + hw->lbk_links;
2999 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
3000 SDP_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
3001 }
3002
3003 /* Set credits for Tx links assuming max packet length allowed.
3004 * This will be reconfigured based on MTU set for PF/VF.
3005 */
3006 for (cgx = 0; cgx < hw->cgx; cgx++) {
3007 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
3008 tx_credits = ((CGX_FIFO_LEN / lmac_cnt) - NIC_HW_MAX_FRS) / 16;
3009 /* Enable credits and set credit pkt count to max allowed */
3010 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
3011 slink = cgx * hw->lmac_per_cgx;
3012 for (link = slink; link < (slink + lmac_cnt); link++) {
3013 rvu_write64(rvu, blkaddr,
3014 NIX_AF_TX_LINKX_NORM_CREDIT(link),
3015 tx_credits);
3016 }
3017 }
3018
3019 /* Set Tx credits for LBK link */
3020 slink = hw->cgx_links;
3021 for (link = slink; link < (slink + hw->lbk_links); link++) {
3022 tx_credits = 1000; /* 10 * max LBK datarate = 10 * 100Gbps */
3023 /* Enable credits and set credit pkt count to max allowed */
3024 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
3025 rvu_write64(rvu, blkaddr,
3026 NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits);
3027 }
3028 }
3029
nix_calibrate_x2p(struct rvu * rvu,int blkaddr)3030 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr)
3031 {
3032 int idx, err;
3033 u64 status;
3034
3035 /* Start X2P bus calibration */
3036 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
3037 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9));
3038 /* Wait for calibration to complete */
3039 err = rvu_poll_reg(rvu, blkaddr,
3040 NIX_AF_STATUS, BIT_ULL(10), false);
3041 if (err) {
3042 dev_err(rvu->dev, "NIX X2P bus calibration failed\n");
3043 return err;
3044 }
3045
3046 status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS);
3047 /* Check if CGX devices are ready */
3048 for (idx = 0; idx < rvu->cgx_cnt_max; idx++) {
3049 /* Skip when cgx port is not available */
3050 if (!rvu_cgx_pdata(idx, rvu) ||
3051 (status & (BIT_ULL(16 + idx))))
3052 continue;
3053 dev_err(rvu->dev,
3054 "CGX%d didn't respond to NIX X2P calibration\n", idx);
3055 err = -EBUSY;
3056 }
3057
3058 /* Check if LBK is ready */
3059 if (!(status & BIT_ULL(19))) {
3060 dev_err(rvu->dev,
3061 "LBK didn't respond to NIX X2P calibration\n");
3062 err = -EBUSY;
3063 }
3064
3065 /* Clear 'calibrate_x2p' bit */
3066 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
3067 rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9));
3068 if (err || (status & 0x3FFULL))
3069 dev_err(rvu->dev,
3070 "NIX X2P calibration failed, status 0x%llx\n", status);
3071 if (err)
3072 return err;
3073 return 0;
3074 }
3075
nix_aq_init(struct rvu * rvu,struct rvu_block * block)3076 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block)
3077 {
3078 u64 cfg;
3079 int err;
3080
3081 /* Set admin queue endianness */
3082 cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG);
3083 #ifdef __BIG_ENDIAN
3084 cfg |= BIT_ULL(8);
3085 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
3086 #else
3087 cfg &= ~BIT_ULL(8);
3088 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
3089 #endif
3090
3091 /* Do not bypass NDC cache */
3092 cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG);
3093 cfg &= ~0x3FFEULL;
3094 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
3095 /* Disable caching of SQB aka SQEs */
3096 cfg |= 0x04ULL;
3097 #endif
3098 rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg);
3099
3100 /* Result structure can be followed by RQ/SQ/CQ context at
3101 * RES + 128bytes and a write mask at RES + 256 bytes, depending on
3102 * operation type. Alloc sufficient result memory for all operations.
3103 */
3104 err = rvu_aq_alloc(rvu, &block->aq,
3105 Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s),
3106 ALIGN(sizeof(struct nix_aq_res_s), 128) + 256);
3107 if (err)
3108 return err;
3109
3110 rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE);
3111 rvu_write64(rvu, block->addr,
3112 NIX_AF_AQ_BASE, (u64)block->aq->inst->iova);
3113 return 0;
3114 }
3115
rvu_nix_init(struct rvu * rvu)3116 int rvu_nix_init(struct rvu *rvu)
3117 {
3118 const struct npc_lt_def_cfg *ltdefs;
3119 struct rvu_hwinfo *hw = rvu->hw;
3120 struct rvu_block *block;
3121 int blkaddr, err;
3122 u64 cfg;
3123
3124 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, 0);
3125 if (blkaddr < 0)
3126 return 0;
3127 block = &hw->block[blkaddr];
3128
3129 if (is_rvu_96xx_B0(rvu)) {
3130 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt
3131 * internal state when conditional clocks are turned off.
3132 * Hence enable them.
3133 */
3134 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
3135 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL);
3136
3137 /* Set chan/link to backpressure TL3 instead of TL2 */
3138 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01);
3139
3140 /* Disable SQ manager's sticky mode operation (set TM6 = 0)
3141 * This sticky mode is known to cause SQ stalls when multiple
3142 * SQs are mapped to same SMQ and transmitting pkts at a time.
3143 */
3144 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS);
3145 cfg &= ~BIT_ULL(15);
3146 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg);
3147 }
3148
3149 ltdefs = rvu->kpu.lt_def;
3150 /* Calibrate X2P bus to check if CGX/LBK links are fine */
3151 err = nix_calibrate_x2p(rvu, blkaddr);
3152 if (err)
3153 return err;
3154
3155 /* Set num of links of each type */
3156 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
3157 hw->cgx = (cfg >> 12) & 0xF;
3158 hw->lmac_per_cgx = (cfg >> 8) & 0xF;
3159 hw->cgx_links = hw->cgx * hw->lmac_per_cgx;
3160 hw->lbk_links = 1;
3161 hw->sdp_links = 1;
3162
3163 /* Initialize admin queue */
3164 err = nix_aq_init(rvu, block);
3165 if (err)
3166 return err;
3167
3168 /* Restore CINT timer delay to HW reset values */
3169 rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL);
3170
3171 if (blkaddr == BLKADDR_NIX0) {
3172 hw->nix0 = devm_kzalloc(rvu->dev,
3173 sizeof(struct nix_hw), GFP_KERNEL);
3174 if (!hw->nix0)
3175 return -ENOMEM;
3176
3177 err = nix_setup_txschq(rvu, hw->nix0, blkaddr);
3178 if (err)
3179 return err;
3180
3181 err = nix_af_mark_format_setup(rvu, hw->nix0, blkaddr);
3182 if (err)
3183 return err;
3184
3185 err = nix_setup_mcast(rvu, hw->nix0, blkaddr);
3186 if (err)
3187 return err;
3188
3189 /* Configure segmentation offload formats */
3190 nix_setup_lso(rvu, hw->nix0, blkaddr);
3191
3192 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info.
3193 * This helps HW protocol checker to identify headers
3194 * and validate length and checksums.
3195 */
3196 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2,
3197 (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) |
3198 ltdefs->rx_ol2.ltype_mask);
3199 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4,
3200 (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) |
3201 ltdefs->rx_oip4.ltype_mask);
3202 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4,
3203 (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) |
3204 ltdefs->rx_iip4.ltype_mask);
3205 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6,
3206 (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) |
3207 ltdefs->rx_oip6.ltype_mask);
3208 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6,
3209 (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) |
3210 ltdefs->rx_iip6.ltype_mask);
3211 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP,
3212 (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) |
3213 ltdefs->rx_otcp.ltype_mask);
3214 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP,
3215 (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) |
3216 ltdefs->rx_itcp.ltype_mask);
3217 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP,
3218 (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) |
3219 ltdefs->rx_oudp.ltype_mask);
3220 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP,
3221 (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) |
3222 ltdefs->rx_iudp.ltype_mask);
3223 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP,
3224 (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) |
3225 ltdefs->rx_osctp.ltype_mask);
3226 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP,
3227 (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) |
3228 ltdefs->rx_isctp.ltype_mask);
3229
3230 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr);
3231 if (err)
3232 return err;
3233
3234 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */
3235 nix_link_config(rvu, blkaddr);
3236
3237 /* Enable Channel backpressure */
3238 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0));
3239 }
3240 return 0;
3241 }
3242
rvu_nix_freemem(struct rvu * rvu)3243 void rvu_nix_freemem(struct rvu *rvu)
3244 {
3245 struct rvu_hwinfo *hw = rvu->hw;
3246 struct rvu_block *block;
3247 struct nix_txsch *txsch;
3248 struct nix_mcast *mcast;
3249 struct nix_hw *nix_hw;
3250 int blkaddr, lvl;
3251
3252 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, 0);
3253 if (blkaddr < 0)
3254 return;
3255
3256 block = &hw->block[blkaddr];
3257 rvu_aq_free(rvu, block->aq);
3258
3259 if (blkaddr == BLKADDR_NIX0) {
3260 nix_hw = get_nix_hw(rvu->hw, blkaddr);
3261 if (!nix_hw)
3262 return;
3263
3264 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
3265 txsch = &nix_hw->txsch[lvl];
3266 kfree(txsch->schq.bmap);
3267 }
3268
3269 mcast = &nix_hw->mcast;
3270 qmem_free(rvu->dev, mcast->mce_ctx);
3271 qmem_free(rvu->dev, mcast->mcast_buf);
3272 mutex_destroy(&mcast->mce_lock);
3273 }
3274 }
3275
rvu_mbox_handler_nix_lf_start_rx(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)3276 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req,
3277 struct msg_rsp *rsp)
3278 {
3279 u16 pcifunc = req->hdr.pcifunc;
3280 int nixlf, err;
3281
3282 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
3283 if (err)
3284 return err;
3285
3286 rvu_npc_enable_default_entries(rvu, pcifunc, nixlf);
3287
3288 return rvu_cgx_start_stop_io(rvu, pcifunc, true);
3289 }
3290
rvu_mbox_handler_nix_lf_stop_rx(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)3291 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req,
3292 struct msg_rsp *rsp)
3293 {
3294 u16 pcifunc = req->hdr.pcifunc;
3295 int nixlf, err;
3296
3297 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
3298 if (err)
3299 return err;
3300
3301 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
3302
3303 return rvu_cgx_start_stop_io(rvu, pcifunc, false);
3304 }
3305
rvu_nix_lf_teardown(struct rvu * rvu,u16 pcifunc,int blkaddr,int nixlf)3306 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
3307 {
3308 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
3309 struct hwctx_disable_req ctx_req;
3310 int err;
3311
3312 ctx_req.hdr.pcifunc = pcifunc;
3313
3314 /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */
3315 nix_interface_deinit(rvu, pcifunc, nixlf);
3316 nix_rx_sync(rvu, blkaddr);
3317 nix_txschq_free(rvu, pcifunc);
3318
3319 rvu_cgx_start_stop_io(rvu, pcifunc, false);
3320
3321 if (pfvf->sq_ctx) {
3322 ctx_req.ctype = NIX_AQ_CTYPE_SQ;
3323 err = nix_lf_hwctx_disable(rvu, &ctx_req);
3324 if (err)
3325 dev_err(rvu->dev, "SQ ctx disable failed\n");
3326 }
3327
3328 if (pfvf->rq_ctx) {
3329 ctx_req.ctype = NIX_AQ_CTYPE_RQ;
3330 err = nix_lf_hwctx_disable(rvu, &ctx_req);
3331 if (err)
3332 dev_err(rvu->dev, "RQ ctx disable failed\n");
3333 }
3334
3335 if (pfvf->cq_ctx) {
3336 ctx_req.ctype = NIX_AQ_CTYPE_CQ;
3337 err = nix_lf_hwctx_disable(rvu, &ctx_req);
3338 if (err)
3339 dev_err(rvu->dev, "CQ ctx disable failed\n");
3340 }
3341
3342 nix_ctx_free(rvu, pfvf);
3343 }
3344
3345 #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32)
3346
rvu_nix_lf_ptp_tx_cfg(struct rvu * rvu,u16 pcifunc,bool enable)3347 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
3348 {
3349 struct rvu_hwinfo *hw = rvu->hw;
3350 struct rvu_block *block;
3351 int blkaddr;
3352 int nixlf;
3353 u64 cfg;
3354
3355 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3356 if (blkaddr < 0)
3357 return NIX_AF_ERR_AF_LF_INVALID;
3358
3359 block = &hw->block[blkaddr];
3360 nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
3361 if (nixlf < 0)
3362 return NIX_AF_ERR_AF_LF_INVALID;
3363
3364 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf));
3365
3366 if (enable)
3367 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN;
3368 else
3369 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN;
3370
3371 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
3372
3373 return 0;
3374 }
3375
rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)3376 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req,
3377 struct msg_rsp *rsp)
3378 {
3379 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true);
3380 }
3381
rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)3382 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req,
3383 struct msg_rsp *rsp)
3384 {
3385 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false);
3386 }
3387
rvu_mbox_handler_nix_lso_format_cfg(struct rvu * rvu,struct nix_lso_format_cfg * req,struct nix_lso_format_cfg_rsp * rsp)3388 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu,
3389 struct nix_lso_format_cfg *req,
3390 struct nix_lso_format_cfg_rsp *rsp)
3391 {
3392 u16 pcifunc = req->hdr.pcifunc;
3393 struct nix_hw *nix_hw;
3394 struct rvu_pfvf *pfvf;
3395 int blkaddr, idx, f;
3396 u64 reg;
3397
3398 pfvf = rvu_get_pfvf(rvu, pcifunc);
3399 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3400 if (!pfvf->nixlf || blkaddr < 0)
3401 return NIX_AF_ERR_AF_LF_INVALID;
3402
3403 nix_hw = get_nix_hw(rvu->hw, blkaddr);
3404 if (!nix_hw)
3405 return -EINVAL;
3406
3407 /* Find existing matching LSO format, if any */
3408 for (idx = 0; idx < nix_hw->lso.in_use; idx++) {
3409 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) {
3410 reg = rvu_read64(rvu, blkaddr,
3411 NIX_AF_LSO_FORMATX_FIELDX(idx, f));
3412 if (req->fields[f] != (reg & req->field_mask))
3413 break;
3414 }
3415
3416 if (f == NIX_LSO_FIELD_MAX)
3417 break;
3418 }
3419
3420 if (idx < nix_hw->lso.in_use) {
3421 /* Match found */
3422 rsp->lso_format_idx = idx;
3423 return 0;
3424 }
3425
3426 if (nix_hw->lso.in_use == nix_hw->lso.total)
3427 return NIX_AF_ERR_LSO_CFG_FAIL;
3428
3429 rsp->lso_format_idx = nix_hw->lso.in_use++;
3430
3431 for (f = 0; f < NIX_LSO_FIELD_MAX; f++)
3432 rvu_write64(rvu, blkaddr,
3433 NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f),
3434 req->fields[f]);
3435
3436 return 0;
3437 }
3438