1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell OcteonTx2 RVU Admin Function driver
3 *
4 * Copyright (C) 2018 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12 #include <linux/interrupt.h>
13 #include <linux/delay.h>
14 #include <linux/irq.h>
15 #include <linux/pci.h>
16 #include <linux/sysfs.h>
17
18 #include "cgx.h"
19 #include "rvu.h"
20 #include "rvu_reg.h"
21 #include "ptp.h"
22
23 #include "rvu_trace.h"
24
25 #define DRV_NAME "octeontx2-af"
26 #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver"
27
28 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
29
30 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
31 struct rvu_block *block, int lf);
32 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
33 struct rvu_block *block, int lf);
34 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
35
36 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
37 int type, int num,
38 void (mbox_handler)(struct work_struct *),
39 void (mbox_up_handler)(struct work_struct *));
40 enum {
41 TYPE_AFVF,
42 TYPE_AFPF,
43 };
44
45 /* Supported devices */
46 static const struct pci_device_id rvu_id_table[] = {
47 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },
48 { 0, } /* end of table */
49 };
50
51 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
52 MODULE_DESCRIPTION(DRV_STRING);
53 MODULE_LICENSE("GPL v2");
54 MODULE_DEVICE_TABLE(pci, rvu_id_table);
55
56 static char *mkex_profile; /* MKEX profile name */
57 module_param(mkex_profile, charp, 0000);
58 MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");
59
rvu_setup_hw_capabilities(struct rvu * rvu)60 static void rvu_setup_hw_capabilities(struct rvu *rvu)
61 {
62 struct rvu_hwinfo *hw = rvu->hw;
63
64 hw->cap.nix_tx_aggr_lvl = NIX_TXSCH_LVL_TL1;
65 hw->cap.nix_fixed_txschq_mapping = false;
66 hw->cap.nix_shaping = true;
67 hw->cap.nix_tx_link_bp = true;
68 hw->cap.nix_rx_multicast = true;
69
70 if (is_rvu_96xx_B0(rvu)) {
71 hw->cap.nix_fixed_txschq_mapping = true;
72 hw->cap.nix_txsch_per_cgx_lmac = 4;
73 hw->cap.nix_txsch_per_lbk_lmac = 132;
74 hw->cap.nix_txsch_per_sdp_lmac = 76;
75 hw->cap.nix_shaping = false;
76 hw->cap.nix_tx_link_bp = false;
77 if (is_rvu_96xx_A0(rvu))
78 hw->cap.nix_rx_multicast = false;
79 }
80 }
81
82 /* Poll a RVU block's register 'offset', for a 'zero'
83 * or 'nonzero' at bits specified by 'mask'
84 */
rvu_poll_reg(struct rvu * rvu,u64 block,u64 offset,u64 mask,bool zero)85 int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)
86 {
87 unsigned long timeout = jiffies + usecs_to_jiffies(20000);
88 bool twice = false;
89 void __iomem *reg;
90 u64 reg_val;
91
92 reg = rvu->afreg_base + ((block << 28) | offset);
93 again:
94 reg_val = readq(reg);
95 if (zero && !(reg_val & mask))
96 return 0;
97 if (!zero && (reg_val & mask))
98 return 0;
99 if (time_before(jiffies, timeout)) {
100 usleep_range(1, 5);
101 goto again;
102 }
103 /* In scenarios where CPU is scheduled out before checking
104 * 'time_before' (above) and gets scheduled in such that
105 * jiffies are beyond timeout value, then check again if HW is
106 * done with the operation in the meantime.
107 */
108 if (!twice) {
109 twice = true;
110 goto again;
111 }
112 return -EBUSY;
113 }
114
rvu_alloc_rsrc(struct rsrc_bmap * rsrc)115 int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)
116 {
117 int id;
118
119 if (!rsrc->bmap)
120 return -EINVAL;
121
122 id = find_first_zero_bit(rsrc->bmap, rsrc->max);
123 if (id >= rsrc->max)
124 return -ENOSPC;
125
126 __set_bit(id, rsrc->bmap);
127
128 return id;
129 }
130
rvu_alloc_rsrc_contig(struct rsrc_bmap * rsrc,int nrsrc)131 int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)
132 {
133 int start;
134
135 if (!rsrc->bmap)
136 return -EINVAL;
137
138 start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
139 if (start >= rsrc->max)
140 return -ENOSPC;
141
142 bitmap_set(rsrc->bmap, start, nrsrc);
143 return start;
144 }
145
rvu_free_rsrc_contig(struct rsrc_bmap * rsrc,int nrsrc,int start)146 static void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)
147 {
148 if (!rsrc->bmap)
149 return;
150 if (start >= rsrc->max)
151 return;
152
153 bitmap_clear(rsrc->bmap, start, nrsrc);
154 }
155
rvu_rsrc_check_contig(struct rsrc_bmap * rsrc,int nrsrc)156 bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)
157 {
158 int start;
159
160 if (!rsrc->bmap)
161 return false;
162
163 start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
164 if (start >= rsrc->max)
165 return false;
166
167 return true;
168 }
169
rvu_free_rsrc(struct rsrc_bmap * rsrc,int id)170 void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)
171 {
172 if (!rsrc->bmap)
173 return;
174
175 __clear_bit(id, rsrc->bmap);
176 }
177
rvu_rsrc_free_count(struct rsrc_bmap * rsrc)178 int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)
179 {
180 int used;
181
182 if (!rsrc->bmap)
183 return 0;
184
185 used = bitmap_weight(rsrc->bmap, rsrc->max);
186 return (rsrc->max - used);
187 }
188
rvu_alloc_bitmap(struct rsrc_bmap * rsrc)189 int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)
190 {
191 rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),
192 sizeof(long), GFP_KERNEL);
193 if (!rsrc->bmap)
194 return -ENOMEM;
195 return 0;
196 }
197
198 /* Get block LF's HW index from a PF_FUNC's block slot number */
rvu_get_lf(struct rvu * rvu,struct rvu_block * block,u16 pcifunc,u16 slot)199 int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)
200 {
201 u16 match = 0;
202 int lf;
203
204 mutex_lock(&rvu->rsrc_lock);
205 for (lf = 0; lf < block->lf.max; lf++) {
206 if (block->fn_map[lf] == pcifunc) {
207 if (slot == match) {
208 mutex_unlock(&rvu->rsrc_lock);
209 return lf;
210 }
211 match++;
212 }
213 }
214 mutex_unlock(&rvu->rsrc_lock);
215 return -ENODEV;
216 }
217
218 /* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.
219 * Some silicon variants of OcteonTX2 supports
220 * multiple blocks of same type.
221 *
222 * @pcifunc has to be zero when no LF is yet attached.
223 */
rvu_get_blkaddr(struct rvu * rvu,int blktype,u16 pcifunc)224 int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)
225 {
226 int devnum, blkaddr = -ENODEV;
227 u64 cfg, reg;
228 bool is_pf;
229
230 switch (blktype) {
231 case BLKTYPE_NPC:
232 blkaddr = BLKADDR_NPC;
233 goto exit;
234 case BLKTYPE_NPA:
235 blkaddr = BLKADDR_NPA;
236 goto exit;
237 case BLKTYPE_NIX:
238 /* For now assume NIX0 */
239 if (!pcifunc) {
240 blkaddr = BLKADDR_NIX0;
241 goto exit;
242 }
243 break;
244 case BLKTYPE_SSO:
245 blkaddr = BLKADDR_SSO;
246 goto exit;
247 case BLKTYPE_SSOW:
248 blkaddr = BLKADDR_SSOW;
249 goto exit;
250 case BLKTYPE_TIM:
251 blkaddr = BLKADDR_TIM;
252 goto exit;
253 case BLKTYPE_CPT:
254 /* For now assume CPT0 */
255 if (!pcifunc) {
256 blkaddr = BLKADDR_CPT0;
257 goto exit;
258 }
259 break;
260 }
261
262 /* Check if this is a RVU PF or VF */
263 if (pcifunc & RVU_PFVF_FUNC_MASK) {
264 is_pf = false;
265 devnum = rvu_get_hwvf(rvu, pcifunc);
266 } else {
267 is_pf = true;
268 devnum = rvu_get_pf(pcifunc);
269 }
270
271 /* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' */
272 if (blktype == BLKTYPE_NIX) {
273 reg = is_pf ? RVU_PRIV_PFX_NIX0_CFG : RVU_PRIV_HWVFX_NIX0_CFG;
274 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
275 if (cfg)
276 blkaddr = BLKADDR_NIX0;
277 }
278
279 /* Check if the 'pcifunc' has a CPT LF from 'BLKADDR_CPT0' */
280 if (blktype == BLKTYPE_CPT) {
281 reg = is_pf ? RVU_PRIV_PFX_CPT0_CFG : RVU_PRIV_HWVFX_CPT0_CFG;
282 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
283 if (cfg)
284 blkaddr = BLKADDR_CPT0;
285 }
286
287 exit:
288 if (is_block_implemented(rvu->hw, blkaddr))
289 return blkaddr;
290 return -ENODEV;
291 }
292
rvu_update_rsrc_map(struct rvu * rvu,struct rvu_pfvf * pfvf,struct rvu_block * block,u16 pcifunc,u16 lf,bool attach)293 static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,
294 struct rvu_block *block, u16 pcifunc,
295 u16 lf, bool attach)
296 {
297 int devnum, num_lfs = 0;
298 bool is_pf;
299 u64 reg;
300
301 if (lf >= block->lf.max) {
302 dev_err(&rvu->pdev->dev,
303 "%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
304 __func__, lf, block->name, block->lf.max);
305 return;
306 }
307
308 /* Check if this is for a RVU PF or VF */
309 if (pcifunc & RVU_PFVF_FUNC_MASK) {
310 is_pf = false;
311 devnum = rvu_get_hwvf(rvu, pcifunc);
312 } else {
313 is_pf = true;
314 devnum = rvu_get_pf(pcifunc);
315 }
316
317 block->fn_map[lf] = attach ? pcifunc : 0;
318
319 switch (block->type) {
320 case BLKTYPE_NPA:
321 pfvf->npalf = attach ? true : false;
322 num_lfs = pfvf->npalf;
323 break;
324 case BLKTYPE_NIX:
325 pfvf->nixlf = attach ? true : false;
326 num_lfs = pfvf->nixlf;
327 break;
328 case BLKTYPE_SSO:
329 attach ? pfvf->sso++ : pfvf->sso--;
330 num_lfs = pfvf->sso;
331 break;
332 case BLKTYPE_SSOW:
333 attach ? pfvf->ssow++ : pfvf->ssow--;
334 num_lfs = pfvf->ssow;
335 break;
336 case BLKTYPE_TIM:
337 attach ? pfvf->timlfs++ : pfvf->timlfs--;
338 num_lfs = pfvf->timlfs;
339 break;
340 case BLKTYPE_CPT:
341 attach ? pfvf->cptlfs++ : pfvf->cptlfs--;
342 num_lfs = pfvf->cptlfs;
343 break;
344 }
345
346 reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;
347 rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);
348 }
349
rvu_get_pf(u16 pcifunc)350 inline int rvu_get_pf(u16 pcifunc)
351 {
352 return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
353 }
354
rvu_get_pf_numvfs(struct rvu * rvu,int pf,int * numvfs,int * hwvf)355 void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
356 {
357 u64 cfg;
358
359 /* Get numVFs attached to this PF and first HWVF */
360 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
361 *numvfs = (cfg >> 12) & 0xFF;
362 *hwvf = cfg & 0xFFF;
363 }
364
rvu_get_hwvf(struct rvu * rvu,int pcifunc)365 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
366 {
367 int pf, func;
368 u64 cfg;
369
370 pf = rvu_get_pf(pcifunc);
371 func = pcifunc & RVU_PFVF_FUNC_MASK;
372
373 /* Get first HWVF attached to this PF */
374 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
375
376 return ((cfg & 0xFFF) + func - 1);
377 }
378
rvu_get_pfvf(struct rvu * rvu,int pcifunc)379 struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
380 {
381 /* Check if it is a PF or VF */
382 if (pcifunc & RVU_PFVF_FUNC_MASK)
383 return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];
384 else
385 return &rvu->pf[rvu_get_pf(pcifunc)];
386 }
387
is_pf_func_valid(struct rvu * rvu,u16 pcifunc)388 static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
389 {
390 int pf, vf, nvfs;
391 u64 cfg;
392
393 pf = rvu_get_pf(pcifunc);
394 if (pf >= rvu->hw->total_pfs)
395 return false;
396
397 if (!(pcifunc & RVU_PFVF_FUNC_MASK))
398 return true;
399
400 /* Check if VF is within number of VFs attached to this PF */
401 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
402 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
403 nvfs = (cfg >> 12) & 0xFF;
404 if (vf >= nvfs)
405 return false;
406
407 return true;
408 }
409
is_block_implemented(struct rvu_hwinfo * hw,int blkaddr)410 bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)
411 {
412 struct rvu_block *block;
413
414 if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)
415 return false;
416
417 block = &hw->block[blkaddr];
418 return block->implemented;
419 }
420
rvu_check_block_implemented(struct rvu * rvu)421 static void rvu_check_block_implemented(struct rvu *rvu)
422 {
423 struct rvu_hwinfo *hw = rvu->hw;
424 struct rvu_block *block;
425 int blkid;
426 u64 cfg;
427
428 /* For each block check if 'implemented' bit is set */
429 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
430 block = &hw->block[blkid];
431 cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));
432 if (cfg & BIT_ULL(11))
433 block->implemented = true;
434 }
435 }
436
rvu_setup_rvum_blk_revid(struct rvu * rvu)437 static void rvu_setup_rvum_blk_revid(struct rvu *rvu)
438 {
439 rvu_write64(rvu, BLKADDR_RVUM,
440 RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM),
441 RVU_BLK_RVUM_REVID);
442 }
443
rvu_clear_rvum_blk_revid(struct rvu * rvu)444 static void rvu_clear_rvum_blk_revid(struct rvu *rvu)
445 {
446 rvu_write64(rvu, BLKADDR_RVUM,
447 RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM), 0x00);
448 }
449
rvu_lf_reset(struct rvu * rvu,struct rvu_block * block,int lf)450 int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)
451 {
452 int err;
453
454 if (!block->implemented)
455 return 0;
456
457 rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));
458 err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),
459 true);
460 return err;
461 }
462
rvu_block_reset(struct rvu * rvu,int blkaddr,u64 rst_reg)463 static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
464 {
465 struct rvu_block *block = &rvu->hw->block[blkaddr];
466
467 if (!block->implemented)
468 return;
469
470 rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
471 rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
472 }
473
rvu_reset_all_blocks(struct rvu * rvu)474 static void rvu_reset_all_blocks(struct rvu *rvu)
475 {
476 /* Do a HW reset of all RVU blocks */
477 rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);
478 rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);
479 rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);
480 rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);
481 rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);
482 rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);
483 rvu_block_reset(rvu, BLKADDR_NDC_NIX0_RX, NDC_AF_BLK_RST);
484 rvu_block_reset(rvu, BLKADDR_NDC_NIX0_TX, NDC_AF_BLK_RST);
485 rvu_block_reset(rvu, BLKADDR_NDC_NPA0, NDC_AF_BLK_RST);
486 }
487
rvu_scan_block(struct rvu * rvu,struct rvu_block * block)488 static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)
489 {
490 struct rvu_pfvf *pfvf;
491 u64 cfg;
492 int lf;
493
494 for (lf = 0; lf < block->lf.max; lf++) {
495 cfg = rvu_read64(rvu, block->addr,
496 block->lfcfg_reg | (lf << block->lfshift));
497 if (!(cfg & BIT_ULL(63)))
498 continue;
499
500 /* Set this resource as being used */
501 __set_bit(lf, block->lf.bmap);
502
503 /* Get, to whom this LF is attached */
504 pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);
505 rvu_update_rsrc_map(rvu, pfvf, block,
506 (cfg >> 8) & 0xFFFF, lf, true);
507
508 /* Set start MSIX vector for this LF within this PF/VF */
509 rvu_set_msix_offset(rvu, pfvf, block, lf);
510 }
511 }
512
rvu_check_min_msix_vec(struct rvu * rvu,int nvecs,int pf,int vf)513 static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
514 {
515 int min_vecs;
516
517 if (!vf)
518 goto check_pf;
519
520 if (!nvecs) {
521 dev_warn(rvu->dev,
522 "PF%d:VF%d is configured with zero msix vectors, %d\n",
523 pf, vf - 1, nvecs);
524 }
525 return;
526
527 check_pf:
528 if (pf == 0)
529 min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;
530 else
531 min_vecs = RVU_PF_INT_VEC_CNT;
532
533 if (!(nvecs < min_vecs))
534 return;
535 dev_warn(rvu->dev,
536 "PF%d is configured with too few vectors, %d, min is %d\n",
537 pf, nvecs, min_vecs);
538 }
539
rvu_setup_msix_resources(struct rvu * rvu)540 static int rvu_setup_msix_resources(struct rvu *rvu)
541 {
542 struct rvu_hwinfo *hw = rvu->hw;
543 int pf, vf, numvfs, hwvf, err;
544 int nvecs, offset, max_msix;
545 struct rvu_pfvf *pfvf;
546 u64 cfg, phy_addr;
547 dma_addr_t iova;
548
549 for (pf = 0; pf < hw->total_pfs; pf++) {
550 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
551 /* If PF is not enabled, nothing to do */
552 if (!((cfg >> 20) & 0x01))
553 continue;
554
555 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
556
557 pfvf = &rvu->pf[pf];
558 /* Get num of MSIX vectors attached to this PF */
559 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));
560 pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;
561 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);
562
563 /* Alloc msix bitmap for this PF */
564 err = rvu_alloc_bitmap(&pfvf->msix);
565 if (err)
566 return err;
567
568 /* Allocate memory for MSIX vector to RVU block LF mapping */
569 pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,
570 sizeof(u16), GFP_KERNEL);
571 if (!pfvf->msix_lfmap)
572 return -ENOMEM;
573
574 /* For PF0 (AF) firmware will set msix vector offsets for
575 * AF, block AF and PF0_INT vectors, so jump to VFs.
576 */
577 if (!pf)
578 goto setup_vfmsix;
579
580 /* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.
581 * These are allocated on driver init and never freed,
582 * so no need to set 'msix_lfmap' for these.
583 */
584 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));
585 nvecs = (cfg >> 12) & 0xFF;
586 cfg &= ~0x7FFULL;
587 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
588 rvu_write64(rvu, BLKADDR_RVUM,
589 RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);
590 setup_vfmsix:
591 /* Alloc msix bitmap for VFs */
592 for (vf = 0; vf < numvfs; vf++) {
593 pfvf = &rvu->hwvf[hwvf + vf];
594 /* Get num of MSIX vectors attached to this VF */
595 cfg = rvu_read64(rvu, BLKADDR_RVUM,
596 RVU_PRIV_PFX_MSIX_CFG(pf));
597 pfvf->msix.max = (cfg & 0xFFF) + 1;
598 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);
599
600 /* Alloc msix bitmap for this VF */
601 err = rvu_alloc_bitmap(&pfvf->msix);
602 if (err)
603 return err;
604
605 pfvf->msix_lfmap =
606 devm_kcalloc(rvu->dev, pfvf->msix.max,
607 sizeof(u16), GFP_KERNEL);
608 if (!pfvf->msix_lfmap)
609 return -ENOMEM;
610
611 /* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.
612 * These are allocated on driver init and never freed,
613 * so no need to set 'msix_lfmap' for these.
614 */
615 cfg = rvu_read64(rvu, BLKADDR_RVUM,
616 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));
617 nvecs = (cfg >> 12) & 0xFF;
618 cfg &= ~0x7FFULL;
619 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
620 rvu_write64(rvu, BLKADDR_RVUM,
621 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),
622 cfg | offset);
623 }
624 }
625
626 /* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence
627 * create a IOMMU mapping for the physcial address configured by
628 * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.
629 */
630 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
631 max_msix = cfg & 0xFFFFF;
632 if (rvu->fwdata && rvu->fwdata->msixtr_base)
633 phy_addr = rvu->fwdata->msixtr_base;
634 else
635 phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);
636
637 iova = dma_map_resource(rvu->dev, phy_addr,
638 max_msix * PCI_MSIX_ENTRY_SIZE,
639 DMA_BIDIRECTIONAL, 0);
640
641 if (dma_mapping_error(rvu->dev, iova))
642 return -ENOMEM;
643
644 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);
645 rvu->msix_base_iova = iova;
646 rvu->msixtr_base_phy = phy_addr;
647
648 return 0;
649 }
650
rvu_reset_msix(struct rvu * rvu)651 static void rvu_reset_msix(struct rvu *rvu)
652 {
653 /* Restore msixtr base register */
654 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE,
655 rvu->msixtr_base_phy);
656 }
657
rvu_free_hw_resources(struct rvu * rvu)658 static void rvu_free_hw_resources(struct rvu *rvu)
659 {
660 struct rvu_hwinfo *hw = rvu->hw;
661 struct rvu_block *block;
662 struct rvu_pfvf *pfvf;
663 int id, max_msix;
664 u64 cfg;
665
666 rvu_npa_freemem(rvu);
667 rvu_npc_freemem(rvu);
668 rvu_nix_freemem(rvu);
669
670 /* Free block LF bitmaps */
671 for (id = 0; id < BLK_COUNT; id++) {
672 block = &hw->block[id];
673 kfree(block->lf.bmap);
674 }
675
676 /* Free MSIX bitmaps */
677 for (id = 0; id < hw->total_pfs; id++) {
678 pfvf = &rvu->pf[id];
679 kfree(pfvf->msix.bmap);
680 }
681
682 for (id = 0; id < hw->total_vfs; id++) {
683 pfvf = &rvu->hwvf[id];
684 kfree(pfvf->msix.bmap);
685 }
686
687 /* Unmap MSIX vector base IOVA mapping */
688 if (!rvu->msix_base_iova)
689 return;
690 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
691 max_msix = cfg & 0xFFFFF;
692 dma_unmap_resource(rvu->dev, rvu->msix_base_iova,
693 max_msix * PCI_MSIX_ENTRY_SIZE,
694 DMA_BIDIRECTIONAL, 0);
695
696 rvu_reset_msix(rvu);
697 mutex_destroy(&rvu->rsrc_lock);
698 }
699
rvu_setup_pfvf_macaddress(struct rvu * rvu)700 static void rvu_setup_pfvf_macaddress(struct rvu *rvu)
701 {
702 struct rvu_hwinfo *hw = rvu->hw;
703 int pf, vf, numvfs, hwvf;
704 struct rvu_pfvf *pfvf;
705 u64 *mac;
706
707 for (pf = 0; pf < hw->total_pfs; pf++) {
708 if (!is_pf_cgxmapped(rvu, pf))
709 continue;
710 /* Assign MAC address to PF */
711 pfvf = &rvu->pf[pf];
712 if (rvu->fwdata && pf < PF_MACNUM_MAX) {
713 mac = &rvu->fwdata->pf_macs[pf];
714 if (*mac)
715 u64_to_ether_addr(*mac, pfvf->mac_addr);
716 else
717 eth_random_addr(pfvf->mac_addr);
718 } else {
719 eth_random_addr(pfvf->mac_addr);
720 }
721
722 /* Assign MAC address to VFs */
723 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
724 for (vf = 0; vf < numvfs; vf++, hwvf++) {
725 pfvf = &rvu->hwvf[hwvf];
726 if (rvu->fwdata && hwvf < VF_MACNUM_MAX) {
727 mac = &rvu->fwdata->vf_macs[hwvf];
728 if (*mac)
729 u64_to_ether_addr(*mac, pfvf->mac_addr);
730 else
731 eth_random_addr(pfvf->mac_addr);
732 } else {
733 eth_random_addr(pfvf->mac_addr);
734 }
735 }
736 }
737 }
738
rvu_fwdata_init(struct rvu * rvu)739 static int rvu_fwdata_init(struct rvu *rvu)
740 {
741 u64 fwdbase;
742 int err;
743
744 /* Get firmware data base address */
745 err = cgx_get_fwdata_base(&fwdbase);
746 if (err)
747 goto fail;
748 rvu->fwdata = ioremap_wc(fwdbase, sizeof(struct rvu_fwdata));
749 if (!rvu->fwdata)
750 goto fail;
751 if (!is_rvu_fwdata_valid(rvu)) {
752 dev_err(rvu->dev,
753 "Mismatch in 'fwdata' struct btw kernel and firmware\n");
754 iounmap(rvu->fwdata);
755 rvu->fwdata = NULL;
756 return -EINVAL;
757 }
758 return 0;
759 fail:
760 dev_info(rvu->dev, "Unable to fetch 'fwdata' from firmware\n");
761 return -EIO;
762 }
763
rvu_fwdata_exit(struct rvu * rvu)764 static void rvu_fwdata_exit(struct rvu *rvu)
765 {
766 if (rvu->fwdata)
767 iounmap(rvu->fwdata);
768 }
769
rvu_setup_hw_resources(struct rvu * rvu)770 static int rvu_setup_hw_resources(struct rvu *rvu)
771 {
772 struct rvu_hwinfo *hw = rvu->hw;
773 struct rvu_block *block;
774 int blkid, err;
775 u64 cfg;
776
777 /* Get HW supported max RVU PF & VF count */
778 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
779 hw->total_pfs = (cfg >> 32) & 0xFF;
780 hw->total_vfs = (cfg >> 20) & 0xFFF;
781 hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;
782
783 /* Init NPA LF's bitmap */
784 block = &hw->block[BLKADDR_NPA];
785 if (!block->implemented)
786 goto nix;
787 cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);
788 block->lf.max = (cfg >> 16) & 0xFFF;
789 block->addr = BLKADDR_NPA;
790 block->type = BLKTYPE_NPA;
791 block->lfshift = 8;
792 block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;
793 block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;
794 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;
795 block->lfcfg_reg = NPA_PRIV_LFX_CFG;
796 block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;
797 block->lfreset_reg = NPA_AF_LF_RST;
798 sprintf(block->name, "NPA");
799 err = rvu_alloc_bitmap(&block->lf);
800 if (err)
801 return err;
802
803 nix:
804 /* Init NIX LF's bitmap */
805 block = &hw->block[BLKADDR_NIX0];
806 if (!block->implemented)
807 goto sso;
808 cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2);
809 block->lf.max = cfg & 0xFFF;
810 block->addr = BLKADDR_NIX0;
811 block->type = BLKTYPE_NIX;
812 block->lfshift = 8;
813 block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;
814 block->pf_lfcnt_reg = RVU_PRIV_PFX_NIX0_CFG;
815 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIX0_CFG;
816 block->lfcfg_reg = NIX_PRIV_LFX_CFG;
817 block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;
818 block->lfreset_reg = NIX_AF_LF_RST;
819 sprintf(block->name, "NIX");
820 err = rvu_alloc_bitmap(&block->lf);
821 if (err)
822 return err;
823
824 sso:
825 /* Init SSO group's bitmap */
826 block = &hw->block[BLKADDR_SSO];
827 if (!block->implemented)
828 goto ssow;
829 cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);
830 block->lf.max = cfg & 0xFFFF;
831 block->addr = BLKADDR_SSO;
832 block->type = BLKTYPE_SSO;
833 block->multislot = true;
834 block->lfshift = 3;
835 block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;
836 block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;
837 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;
838 block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;
839 block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;
840 block->lfreset_reg = SSO_AF_LF_HWGRP_RST;
841 sprintf(block->name, "SSO GROUP");
842 err = rvu_alloc_bitmap(&block->lf);
843 if (err)
844 return err;
845
846 ssow:
847 /* Init SSO workslot's bitmap */
848 block = &hw->block[BLKADDR_SSOW];
849 if (!block->implemented)
850 goto tim;
851 block->lf.max = (cfg >> 56) & 0xFF;
852 block->addr = BLKADDR_SSOW;
853 block->type = BLKTYPE_SSOW;
854 block->multislot = true;
855 block->lfshift = 3;
856 block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;
857 block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;
858 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;
859 block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;
860 block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;
861 block->lfreset_reg = SSOW_AF_LF_HWS_RST;
862 sprintf(block->name, "SSOWS");
863 err = rvu_alloc_bitmap(&block->lf);
864 if (err)
865 return err;
866
867 tim:
868 /* Init TIM LF's bitmap */
869 block = &hw->block[BLKADDR_TIM];
870 if (!block->implemented)
871 goto cpt;
872 cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);
873 block->lf.max = cfg & 0xFFFF;
874 block->addr = BLKADDR_TIM;
875 block->type = BLKTYPE_TIM;
876 block->multislot = true;
877 block->lfshift = 3;
878 block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;
879 block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;
880 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;
881 block->lfcfg_reg = TIM_PRIV_LFX_CFG;
882 block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;
883 block->lfreset_reg = TIM_AF_LF_RST;
884 sprintf(block->name, "TIM");
885 err = rvu_alloc_bitmap(&block->lf);
886 if (err)
887 return err;
888
889 cpt:
890 /* Init CPT LF's bitmap */
891 block = &hw->block[BLKADDR_CPT0];
892 if (!block->implemented)
893 goto init;
894 cfg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS0);
895 block->lf.max = cfg & 0xFF;
896 block->addr = BLKADDR_CPT0;
897 block->type = BLKTYPE_CPT;
898 block->multislot = true;
899 block->lfshift = 3;
900 block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;
901 block->pf_lfcnt_reg = RVU_PRIV_PFX_CPT0_CFG;
902 block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPT0_CFG;
903 block->lfcfg_reg = CPT_PRIV_LFX_CFG;
904 block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;
905 block->lfreset_reg = CPT_AF_LF_RST;
906 sprintf(block->name, "CPT");
907 err = rvu_alloc_bitmap(&block->lf);
908 if (err)
909 return err;
910
911 init:
912 /* Allocate memory for PFVF data */
913 rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,
914 sizeof(struct rvu_pfvf), GFP_KERNEL);
915 if (!rvu->pf)
916 return -ENOMEM;
917
918 rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,
919 sizeof(struct rvu_pfvf), GFP_KERNEL);
920 if (!rvu->hwvf)
921 return -ENOMEM;
922
923 mutex_init(&rvu->rsrc_lock);
924
925 rvu_fwdata_init(rvu);
926
927 err = rvu_setup_msix_resources(rvu);
928 if (err)
929 return err;
930
931 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
932 block = &hw->block[blkid];
933 if (!block->lf.bmap)
934 continue;
935
936 /* Allocate memory for block LF/slot to pcifunc mapping info */
937 block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,
938 sizeof(u16), GFP_KERNEL);
939 if (!block->fn_map) {
940 err = -ENOMEM;
941 goto msix_err;
942 }
943
944 /* Scan all blocks to check if low level firmware has
945 * already provisioned any of the resources to a PF/VF.
946 */
947 rvu_scan_block(rvu, block);
948 }
949
950 err = rvu_npc_init(rvu);
951 if (err)
952 goto npc_err;
953
954 err = rvu_cgx_init(rvu);
955 if (err)
956 goto cgx_err;
957
958 /* Assign MACs for CGX mapped functions */
959 rvu_setup_pfvf_macaddress(rvu);
960
961 err = rvu_npa_init(rvu);
962 if (err)
963 goto npa_err;
964
965 err = rvu_nix_init(rvu);
966 if (err)
967 goto nix_err;
968
969 return 0;
970
971 nix_err:
972 rvu_nix_freemem(rvu);
973 npa_err:
974 rvu_npa_freemem(rvu);
975 cgx_err:
976 rvu_cgx_exit(rvu);
977 npc_err:
978 rvu_npc_freemem(rvu);
979 rvu_fwdata_exit(rvu);
980 msix_err:
981 rvu_reset_msix(rvu);
982 return err;
983 }
984
985 /* NPA and NIX admin queue APIs */
rvu_aq_free(struct rvu * rvu,struct admin_queue * aq)986 void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)
987 {
988 if (!aq)
989 return;
990
991 qmem_free(rvu->dev, aq->inst);
992 qmem_free(rvu->dev, aq->res);
993 devm_kfree(rvu->dev, aq);
994 }
995
rvu_aq_alloc(struct rvu * rvu,struct admin_queue ** ad_queue,int qsize,int inst_size,int res_size)996 int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,
997 int qsize, int inst_size, int res_size)
998 {
999 struct admin_queue *aq;
1000 int err;
1001
1002 *ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);
1003 if (!*ad_queue)
1004 return -ENOMEM;
1005 aq = *ad_queue;
1006
1007 /* Alloc memory for instructions i.e AQ */
1008 err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);
1009 if (err) {
1010 devm_kfree(rvu->dev, aq);
1011 return err;
1012 }
1013
1014 /* Alloc memory for results */
1015 err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);
1016 if (err) {
1017 rvu_aq_free(rvu, aq);
1018 return err;
1019 }
1020
1021 spin_lock_init(&aq->lock);
1022 return 0;
1023 }
1024
rvu_mbox_handler_ready(struct rvu * rvu,struct msg_req * req,struct ready_msg_rsp * rsp)1025 int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,
1026 struct ready_msg_rsp *rsp)
1027 {
1028 if (rvu->fwdata) {
1029 rsp->rclk_freq = rvu->fwdata->rclk;
1030 rsp->sclk_freq = rvu->fwdata->sclk;
1031 }
1032 return 0;
1033 }
1034
1035 /* Get current count of a RVU block's LF/slots
1036 * provisioned to a given RVU func.
1037 */
rvu_get_rsrc_mapcount(struct rvu_pfvf * pfvf,int blktype)1038 static u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blktype)
1039 {
1040 switch (blktype) {
1041 case BLKTYPE_NPA:
1042 return pfvf->npalf ? 1 : 0;
1043 case BLKTYPE_NIX:
1044 return pfvf->nixlf ? 1 : 0;
1045 case BLKTYPE_SSO:
1046 return pfvf->sso;
1047 case BLKTYPE_SSOW:
1048 return pfvf->ssow;
1049 case BLKTYPE_TIM:
1050 return pfvf->timlfs;
1051 case BLKTYPE_CPT:
1052 return pfvf->cptlfs;
1053 }
1054 return 0;
1055 }
1056
is_pffunc_map_valid(struct rvu * rvu,u16 pcifunc,int blktype)1057 bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)
1058 {
1059 struct rvu_pfvf *pfvf;
1060
1061 if (!is_pf_func_valid(rvu, pcifunc))
1062 return false;
1063
1064 pfvf = rvu_get_pfvf(rvu, pcifunc);
1065
1066 /* Check if this PFFUNC has a LF of type blktype attached */
1067 if (!rvu_get_rsrc_mapcount(pfvf, blktype))
1068 return false;
1069
1070 return true;
1071 }
1072
rvu_lookup_rsrc(struct rvu * rvu,struct rvu_block * block,int pcifunc,int slot)1073 static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,
1074 int pcifunc, int slot)
1075 {
1076 u64 val;
1077
1078 val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);
1079 rvu_write64(rvu, block->addr, block->lookup_reg, val);
1080 /* Wait for the lookup to finish */
1081 /* TODO: put some timeout here */
1082 while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))
1083 ;
1084
1085 val = rvu_read64(rvu, block->addr, block->lookup_reg);
1086
1087 /* Check LF valid bit */
1088 if (!(val & (1ULL << 12)))
1089 return -1;
1090
1091 return (val & 0xFFF);
1092 }
1093
rvu_detach_block(struct rvu * rvu,int pcifunc,int blktype)1094 static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
1095 {
1096 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1097 struct rvu_hwinfo *hw = rvu->hw;
1098 struct rvu_block *block;
1099 int slot, lf, num_lfs;
1100 int blkaddr;
1101
1102 blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);
1103 if (blkaddr < 0)
1104 return;
1105
1106 block = &hw->block[blkaddr];
1107
1108 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1109 if (!num_lfs)
1110 return;
1111
1112 for (slot = 0; slot < num_lfs; slot++) {
1113 lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);
1114 if (lf < 0) /* This should never happen */
1115 continue;
1116
1117 /* Disable the LF */
1118 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1119 (lf << block->lfshift), 0x00ULL);
1120
1121 /* Update SW maintained mapping info as well */
1122 rvu_update_rsrc_map(rvu, pfvf, block,
1123 pcifunc, lf, false);
1124
1125 /* Free the resource */
1126 rvu_free_rsrc(&block->lf, lf);
1127
1128 /* Clear MSIX vector offset for this LF */
1129 rvu_clear_msix_offset(rvu, pfvf, block, lf);
1130 }
1131 }
1132
rvu_detach_rsrcs(struct rvu * rvu,struct rsrc_detach * detach,u16 pcifunc)1133 static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
1134 u16 pcifunc)
1135 {
1136 struct rvu_hwinfo *hw = rvu->hw;
1137 bool detach_all = true;
1138 struct rvu_block *block;
1139 int blkid;
1140
1141 mutex_lock(&rvu->rsrc_lock);
1142
1143 /* Check for partial resource detach */
1144 if (detach && detach->partial)
1145 detach_all = false;
1146
1147 /* Check for RVU block's LFs attached to this func,
1148 * if so, detach them.
1149 */
1150 for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1151 block = &hw->block[blkid];
1152 if (!block->lf.bmap)
1153 continue;
1154 if (!detach_all && detach) {
1155 if (blkid == BLKADDR_NPA && !detach->npalf)
1156 continue;
1157 else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)
1158 continue;
1159 else if ((blkid == BLKADDR_SSO) && !detach->sso)
1160 continue;
1161 else if ((blkid == BLKADDR_SSOW) && !detach->ssow)
1162 continue;
1163 else if ((blkid == BLKADDR_TIM) && !detach->timlfs)
1164 continue;
1165 else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)
1166 continue;
1167 }
1168 rvu_detach_block(rvu, pcifunc, block->type);
1169 }
1170
1171 mutex_unlock(&rvu->rsrc_lock);
1172 return 0;
1173 }
1174
rvu_mbox_handler_detach_resources(struct rvu * rvu,struct rsrc_detach * detach,struct msg_rsp * rsp)1175 int rvu_mbox_handler_detach_resources(struct rvu *rvu,
1176 struct rsrc_detach *detach,
1177 struct msg_rsp *rsp)
1178 {
1179 return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);
1180 }
1181
rvu_attach_block(struct rvu * rvu,int pcifunc,int blktype,int num_lfs)1182 static void rvu_attach_block(struct rvu *rvu, int pcifunc,
1183 int blktype, int num_lfs)
1184 {
1185 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1186 struct rvu_hwinfo *hw = rvu->hw;
1187 struct rvu_block *block;
1188 int slot, lf;
1189 int blkaddr;
1190 u64 cfg;
1191
1192 if (!num_lfs)
1193 return;
1194
1195 blkaddr = rvu_get_blkaddr(rvu, blktype, 0);
1196 if (blkaddr < 0)
1197 return;
1198
1199 block = &hw->block[blkaddr];
1200 if (!block->lf.bmap)
1201 return;
1202
1203 for (slot = 0; slot < num_lfs; slot++) {
1204 /* Allocate the resource */
1205 lf = rvu_alloc_rsrc(&block->lf);
1206 if (lf < 0)
1207 return;
1208
1209 cfg = (1ULL << 63) | (pcifunc << 8) | slot;
1210 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1211 (lf << block->lfshift), cfg);
1212 rvu_update_rsrc_map(rvu, pfvf, block,
1213 pcifunc, lf, true);
1214
1215 /* Set start MSIX vector for this LF within this PF/VF */
1216 rvu_set_msix_offset(rvu, pfvf, block, lf);
1217 }
1218 }
1219
rvu_check_rsrc_availability(struct rvu * rvu,struct rsrc_attach * req,u16 pcifunc)1220 static int rvu_check_rsrc_availability(struct rvu *rvu,
1221 struct rsrc_attach *req, u16 pcifunc)
1222 {
1223 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1224 struct rvu_hwinfo *hw = rvu->hw;
1225 struct rvu_block *block;
1226 int free_lfs, mappedlfs;
1227
1228 /* Only one NPA LF can be attached */
1229 if (req->npalf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NPA)) {
1230 block = &hw->block[BLKADDR_NPA];
1231 free_lfs = rvu_rsrc_free_count(&block->lf);
1232 if (!free_lfs)
1233 goto fail;
1234 } else if (req->npalf) {
1235 dev_err(&rvu->pdev->dev,
1236 "Func 0x%x: Invalid req, already has NPA\n",
1237 pcifunc);
1238 return -EINVAL;
1239 }
1240
1241 /* Only one NIX LF can be attached */
1242 if (req->nixlf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NIX)) {
1243 block = &hw->block[BLKADDR_NIX0];
1244 free_lfs = rvu_rsrc_free_count(&block->lf);
1245 if (!free_lfs)
1246 goto fail;
1247 } else if (req->nixlf) {
1248 dev_err(&rvu->pdev->dev,
1249 "Func 0x%x: Invalid req, already has NIX\n",
1250 pcifunc);
1251 return -EINVAL;
1252 }
1253
1254 if (req->sso) {
1255 block = &hw->block[BLKADDR_SSO];
1256 /* Is request within limits ? */
1257 if (req->sso > block->lf.max) {
1258 dev_err(&rvu->pdev->dev,
1259 "Func 0x%x: Invalid SSO req, %d > max %d\n",
1260 pcifunc, req->sso, block->lf.max);
1261 return -EINVAL;
1262 }
1263 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1264 free_lfs = rvu_rsrc_free_count(&block->lf);
1265 /* Check if additional resources are available */
1266 if (req->sso > mappedlfs &&
1267 ((req->sso - mappedlfs) > free_lfs))
1268 goto fail;
1269 }
1270
1271 if (req->ssow) {
1272 block = &hw->block[BLKADDR_SSOW];
1273 if (req->ssow > block->lf.max) {
1274 dev_err(&rvu->pdev->dev,
1275 "Func 0x%x: Invalid SSOW req, %d > max %d\n",
1276 pcifunc, req->sso, block->lf.max);
1277 return -EINVAL;
1278 }
1279 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1280 free_lfs = rvu_rsrc_free_count(&block->lf);
1281 if (req->ssow > mappedlfs &&
1282 ((req->ssow - mappedlfs) > free_lfs))
1283 goto fail;
1284 }
1285
1286 if (req->timlfs) {
1287 block = &hw->block[BLKADDR_TIM];
1288 if (req->timlfs > block->lf.max) {
1289 dev_err(&rvu->pdev->dev,
1290 "Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1291 pcifunc, req->timlfs, block->lf.max);
1292 return -EINVAL;
1293 }
1294 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1295 free_lfs = rvu_rsrc_free_count(&block->lf);
1296 if (req->timlfs > mappedlfs &&
1297 ((req->timlfs - mappedlfs) > free_lfs))
1298 goto fail;
1299 }
1300
1301 if (req->cptlfs) {
1302 block = &hw->block[BLKADDR_CPT0];
1303 if (req->cptlfs > block->lf.max) {
1304 dev_err(&rvu->pdev->dev,
1305 "Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1306 pcifunc, req->cptlfs, block->lf.max);
1307 return -EINVAL;
1308 }
1309 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1310 free_lfs = rvu_rsrc_free_count(&block->lf);
1311 if (req->cptlfs > mappedlfs &&
1312 ((req->cptlfs - mappedlfs) > free_lfs))
1313 goto fail;
1314 }
1315
1316 return 0;
1317
1318 fail:
1319 dev_info(rvu->dev, "Request for %s failed\n", block->name);
1320 return -ENOSPC;
1321 }
1322
rvu_mbox_handler_attach_resources(struct rvu * rvu,struct rsrc_attach * attach,struct msg_rsp * rsp)1323 int rvu_mbox_handler_attach_resources(struct rvu *rvu,
1324 struct rsrc_attach *attach,
1325 struct msg_rsp *rsp)
1326 {
1327 u16 pcifunc = attach->hdr.pcifunc;
1328 int err;
1329
1330 /* If first request, detach all existing attached resources */
1331 if (!attach->modify)
1332 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1333
1334 mutex_lock(&rvu->rsrc_lock);
1335
1336 /* Check if the request can be accommodated */
1337 err = rvu_check_rsrc_availability(rvu, attach, pcifunc);
1338 if (err)
1339 goto exit;
1340
1341 /* Now attach the requested resources */
1342 if (attach->npalf)
1343 rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1);
1344
1345 if (attach->nixlf)
1346 rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1);
1347
1348 if (attach->sso) {
1349 /* RVU func doesn't know which exact LF or slot is attached
1350 * to it, it always sees as slot 0,1,2. So for a 'modify'
1351 * request, simply detach all existing attached LFs/slots
1352 * and attach a fresh.
1353 */
1354 if (attach->modify)
1355 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1356 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso);
1357 }
1358
1359 if (attach->ssow) {
1360 if (attach->modify)
1361 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1362 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow);
1363 }
1364
1365 if (attach->timlfs) {
1366 if (attach->modify)
1367 rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1368 rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs);
1369 }
1370
1371 if (attach->cptlfs) {
1372 if (attach->modify)
1373 rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);
1374 rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs);
1375 }
1376
1377 exit:
1378 mutex_unlock(&rvu->rsrc_lock);
1379 return err;
1380 }
1381
rvu_get_msix_offset(struct rvu * rvu,struct rvu_pfvf * pfvf,int blkaddr,int lf)1382 static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1383 int blkaddr, int lf)
1384 {
1385 u16 vec;
1386
1387 if (lf < 0)
1388 return MSIX_VECTOR_INVALID;
1389
1390 for (vec = 0; vec < pfvf->msix.max; vec++) {
1391 if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))
1392 return vec;
1393 }
1394 return MSIX_VECTOR_INVALID;
1395 }
1396
rvu_set_msix_offset(struct rvu * rvu,struct rvu_pfvf * pfvf,struct rvu_block * block,int lf)1397 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1398 struct rvu_block *block, int lf)
1399 {
1400 u16 nvecs, vec, offset;
1401 u64 cfg;
1402
1403 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1404 (lf << block->lfshift));
1405 nvecs = (cfg >> 12) & 0xFF;
1406
1407 /* Check and alloc MSIX vectors, must be contiguous */
1408 if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))
1409 return;
1410
1411 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
1412
1413 /* Config MSIX offset in LF */
1414 rvu_write64(rvu, block->addr, block->msixcfg_reg |
1415 (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);
1416
1417 /* Update the bitmap as well */
1418 for (vec = 0; vec < nvecs; vec++)
1419 pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);
1420 }
1421
rvu_clear_msix_offset(struct rvu * rvu,struct rvu_pfvf * pfvf,struct rvu_block * block,int lf)1422 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1423 struct rvu_block *block, int lf)
1424 {
1425 u16 nvecs, vec, offset;
1426 u64 cfg;
1427
1428 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1429 (lf << block->lfshift));
1430 nvecs = (cfg >> 12) & 0xFF;
1431
1432 /* Clear MSIX offset in LF */
1433 rvu_write64(rvu, block->addr, block->msixcfg_reg |
1434 (lf << block->lfshift), cfg & ~0x7FFULL);
1435
1436 offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);
1437
1438 /* Update the mapping */
1439 for (vec = 0; vec < nvecs; vec++)
1440 pfvf->msix_lfmap[offset + vec] = 0;
1441
1442 /* Free the same in MSIX bitmap */
1443 rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);
1444 }
1445
rvu_mbox_handler_msix_offset(struct rvu * rvu,struct msg_req * req,struct msix_offset_rsp * rsp)1446 int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,
1447 struct msix_offset_rsp *rsp)
1448 {
1449 struct rvu_hwinfo *hw = rvu->hw;
1450 u16 pcifunc = req->hdr.pcifunc;
1451 struct rvu_pfvf *pfvf;
1452 int lf, slot;
1453
1454 pfvf = rvu_get_pfvf(rvu, pcifunc);
1455 if (!pfvf->msix.bmap)
1456 return 0;
1457
1458 /* Set MSIX offsets for each block's LFs attached to this PF/VF */
1459 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);
1460 rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);
1461
1462 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NIX0], pcifunc, 0);
1463 rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NIX0, lf);
1464
1465 rsp->sso = pfvf->sso;
1466 for (slot = 0; slot < rsp->sso; slot++) {
1467 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);
1468 rsp->sso_msixoff[slot] =
1469 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);
1470 }
1471
1472 rsp->ssow = pfvf->ssow;
1473 for (slot = 0; slot < rsp->ssow; slot++) {
1474 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);
1475 rsp->ssow_msixoff[slot] =
1476 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);
1477 }
1478
1479 rsp->timlfs = pfvf->timlfs;
1480 for (slot = 0; slot < rsp->timlfs; slot++) {
1481 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);
1482 rsp->timlf_msixoff[slot] =
1483 rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);
1484 }
1485
1486 rsp->cptlfs = pfvf->cptlfs;
1487 for (slot = 0; slot < rsp->cptlfs; slot++) {
1488 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);
1489 rsp->cptlf_msixoff[slot] =
1490 rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);
1491 }
1492 return 0;
1493 }
1494
rvu_mbox_handler_vf_flr(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)1495 int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,
1496 struct msg_rsp *rsp)
1497 {
1498 u16 pcifunc = req->hdr.pcifunc;
1499 u16 vf, numvfs;
1500 u64 cfg;
1501
1502 vf = pcifunc & RVU_PFVF_FUNC_MASK;
1503 cfg = rvu_read64(rvu, BLKADDR_RVUM,
1504 RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc)));
1505 numvfs = (cfg >> 12) & 0xFF;
1506
1507 if (vf && vf <= numvfs)
1508 __rvu_flr_handler(rvu, pcifunc);
1509 else
1510 return RVU_INVALID_VF_ID;
1511
1512 return 0;
1513 }
1514
rvu_mbox_handler_get_hw_cap(struct rvu * rvu,struct msg_req * req,struct get_hw_cap_rsp * rsp)1515 int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req,
1516 struct get_hw_cap_rsp *rsp)
1517 {
1518 struct rvu_hwinfo *hw = rvu->hw;
1519
1520 rsp->nix_fixed_txschq_mapping = hw->cap.nix_fixed_txschq_mapping;
1521 rsp->nix_shaping = hw->cap.nix_shaping;
1522
1523 return 0;
1524 }
1525
rvu_process_mbox_msg(struct otx2_mbox * mbox,int devid,struct mbox_msghdr * req)1526 static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
1527 struct mbox_msghdr *req)
1528 {
1529 struct rvu *rvu = pci_get_drvdata(mbox->pdev);
1530
1531 /* Check if valid, if not reply with a invalid msg */
1532 if (req->sig != OTX2_MBOX_REQ_SIG)
1533 goto bad_message;
1534
1535 switch (req->id) {
1536 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
1537 case _id: { \
1538 struct _rsp_type *rsp; \
1539 int err; \
1540 \
1541 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
1542 mbox, devid, \
1543 sizeof(struct _rsp_type)); \
1544 /* some handlers should complete even if reply */ \
1545 /* could not be allocated */ \
1546 if (!rsp && \
1547 _id != MBOX_MSG_DETACH_RESOURCES && \
1548 _id != MBOX_MSG_NIX_TXSCH_FREE && \
1549 _id != MBOX_MSG_VF_FLR) \
1550 return -ENOMEM; \
1551 if (rsp) { \
1552 rsp->hdr.id = _id; \
1553 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
1554 rsp->hdr.pcifunc = req->pcifunc; \
1555 rsp->hdr.rc = 0; \
1556 } \
1557 \
1558 err = rvu_mbox_handler_ ## _fn_name(rvu, \
1559 (struct _req_type *)req, \
1560 rsp); \
1561 if (rsp && err) \
1562 rsp->hdr.rc = err; \
1563 \
1564 trace_otx2_msg_process(mbox->pdev, _id, err); \
1565 return rsp ? err : -ENOMEM; \
1566 }
1567 MBOX_MESSAGES
1568 #undef M
1569
1570 bad_message:
1571 default:
1572 otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);
1573 return -ENODEV;
1574 }
1575 }
1576
__rvu_mbox_handler(struct rvu_work * mwork,int type)1577 static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
1578 {
1579 struct rvu *rvu = mwork->rvu;
1580 int offset, err, id, devid;
1581 struct otx2_mbox_dev *mdev;
1582 struct mbox_hdr *req_hdr;
1583 struct mbox_msghdr *msg;
1584 struct mbox_wq_info *mw;
1585 struct otx2_mbox *mbox;
1586
1587 switch (type) {
1588 case TYPE_AFPF:
1589 mw = &rvu->afpf_wq_info;
1590 break;
1591 case TYPE_AFVF:
1592 mw = &rvu->afvf_wq_info;
1593 break;
1594 default:
1595 return;
1596 }
1597
1598 devid = mwork - mw->mbox_wrk;
1599 mbox = &mw->mbox;
1600 mdev = &mbox->dev[devid];
1601
1602 /* Process received mbox messages */
1603 req_hdr = mdev->mbase + mbox->rx_start;
1604 if (mw->mbox_wrk[devid].num_msgs == 0)
1605 return;
1606
1607 offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
1608
1609 for (id = 0; id < mw->mbox_wrk[devid].num_msgs; id++) {
1610 msg = mdev->mbase + offset;
1611
1612 /* Set which PF/VF sent this message based on mbox IRQ */
1613 switch (type) {
1614 case TYPE_AFPF:
1615 msg->pcifunc &=
1616 ~(RVU_PFVF_PF_MASK << RVU_PFVF_PF_SHIFT);
1617 msg->pcifunc |= (devid << RVU_PFVF_PF_SHIFT);
1618 break;
1619 case TYPE_AFVF:
1620 msg->pcifunc &=
1621 ~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);
1622 msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;
1623 break;
1624 }
1625
1626 err = rvu_process_mbox_msg(mbox, devid, msg);
1627 if (!err) {
1628 offset = mbox->rx_start + msg->next_msgoff;
1629 continue;
1630 }
1631
1632 if (msg->pcifunc & RVU_PFVF_FUNC_MASK)
1633 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
1634 err, otx2_mbox_id2name(msg->id),
1635 msg->id, rvu_get_pf(msg->pcifunc),
1636 (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);
1637 else
1638 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",
1639 err, otx2_mbox_id2name(msg->id),
1640 msg->id, devid);
1641 }
1642 mw->mbox_wrk[devid].num_msgs = 0;
1643
1644 /* Send mbox responses to VF/PF */
1645 otx2_mbox_msg_send(mbox, devid);
1646 }
1647
rvu_afpf_mbox_handler(struct work_struct * work)1648 static inline void rvu_afpf_mbox_handler(struct work_struct *work)
1649 {
1650 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1651
1652 __rvu_mbox_handler(mwork, TYPE_AFPF);
1653 }
1654
rvu_afvf_mbox_handler(struct work_struct * work)1655 static inline void rvu_afvf_mbox_handler(struct work_struct *work)
1656 {
1657 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1658
1659 __rvu_mbox_handler(mwork, TYPE_AFVF);
1660 }
1661
__rvu_mbox_up_handler(struct rvu_work * mwork,int type)1662 static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
1663 {
1664 struct rvu *rvu = mwork->rvu;
1665 struct otx2_mbox_dev *mdev;
1666 struct mbox_hdr *rsp_hdr;
1667 struct mbox_msghdr *msg;
1668 struct mbox_wq_info *mw;
1669 struct otx2_mbox *mbox;
1670 int offset, id, devid;
1671
1672 switch (type) {
1673 case TYPE_AFPF:
1674 mw = &rvu->afpf_wq_info;
1675 break;
1676 case TYPE_AFVF:
1677 mw = &rvu->afvf_wq_info;
1678 break;
1679 default:
1680 return;
1681 }
1682
1683 devid = mwork - mw->mbox_wrk_up;
1684 mbox = &mw->mbox_up;
1685 mdev = &mbox->dev[devid];
1686
1687 rsp_hdr = mdev->mbase + mbox->rx_start;
1688 if (mw->mbox_wrk_up[devid].up_num_msgs == 0) {
1689 dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");
1690 return;
1691 }
1692
1693 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1694
1695 for (id = 0; id < mw->mbox_wrk_up[devid].up_num_msgs; id++) {
1696 msg = mdev->mbase + offset;
1697
1698 if (msg->id >= MBOX_MSG_MAX) {
1699 dev_err(rvu->dev,
1700 "Mbox msg with unknown ID 0x%x\n", msg->id);
1701 goto end;
1702 }
1703
1704 if (msg->sig != OTX2_MBOX_RSP_SIG) {
1705 dev_err(rvu->dev,
1706 "Mbox msg with wrong signature %x, ID 0x%x\n",
1707 msg->sig, msg->id);
1708 goto end;
1709 }
1710
1711 switch (msg->id) {
1712 case MBOX_MSG_CGX_LINK_EVENT:
1713 break;
1714 default:
1715 if (msg->rc)
1716 dev_err(rvu->dev,
1717 "Mbox msg response has err %d, ID 0x%x\n",
1718 msg->rc, msg->id);
1719 break;
1720 }
1721 end:
1722 offset = mbox->rx_start + msg->next_msgoff;
1723 mdev->msgs_acked++;
1724 }
1725 mw->mbox_wrk_up[devid].up_num_msgs = 0;
1726
1727 otx2_mbox_reset(mbox, devid);
1728 }
1729
rvu_afpf_mbox_up_handler(struct work_struct * work)1730 static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)
1731 {
1732 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1733
1734 __rvu_mbox_up_handler(mwork, TYPE_AFPF);
1735 }
1736
rvu_afvf_mbox_up_handler(struct work_struct * work)1737 static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
1738 {
1739 struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1740
1741 __rvu_mbox_up_handler(mwork, TYPE_AFVF);
1742 }
1743
rvu_mbox_init(struct rvu * rvu,struct mbox_wq_info * mw,int type,int num,void (mbox_handler)(struct work_struct *),void (mbox_up_handler)(struct work_struct *))1744 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
1745 int type, int num,
1746 void (mbox_handler)(struct work_struct *),
1747 void (mbox_up_handler)(struct work_struct *))
1748 {
1749 void __iomem *hwbase = NULL, *reg_base;
1750 int err, i, dir, dir_up;
1751 struct rvu_work *mwork;
1752 const char *name;
1753 u64 bar4_addr;
1754
1755 switch (type) {
1756 case TYPE_AFPF:
1757 name = "rvu_afpf_mailbox";
1758 bar4_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PF_BAR4_ADDR);
1759 dir = MBOX_DIR_AFPF;
1760 dir_up = MBOX_DIR_AFPF_UP;
1761 reg_base = rvu->afreg_base;
1762 break;
1763 case TYPE_AFVF:
1764 name = "rvu_afvf_mailbox";
1765 bar4_addr = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
1766 dir = MBOX_DIR_PFVF;
1767 dir_up = MBOX_DIR_PFVF_UP;
1768 reg_base = rvu->pfreg_base;
1769 break;
1770 default:
1771 return -EINVAL;
1772 }
1773
1774 mw->mbox_wq = alloc_workqueue(name,
1775 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
1776 num);
1777 if (!mw->mbox_wq)
1778 return -ENOMEM;
1779
1780 mw->mbox_wrk = devm_kcalloc(rvu->dev, num,
1781 sizeof(struct rvu_work), GFP_KERNEL);
1782 if (!mw->mbox_wrk) {
1783 err = -ENOMEM;
1784 goto exit;
1785 }
1786
1787 mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,
1788 sizeof(struct rvu_work), GFP_KERNEL);
1789 if (!mw->mbox_wrk_up) {
1790 err = -ENOMEM;
1791 goto exit;
1792 }
1793
1794 /* Mailbox is a reserved memory (in RAM) region shared between
1795 * RVU devices, shouldn't be mapped as device memory to allow
1796 * unaligned accesses.
1797 */
1798 hwbase = ioremap_wc(bar4_addr, MBOX_SIZE * num);
1799 if (!hwbase) {
1800 dev_err(rvu->dev, "Unable to map mailbox region\n");
1801 err = -ENOMEM;
1802 goto exit;
1803 }
1804
1805 err = otx2_mbox_init(&mw->mbox, hwbase, rvu->pdev, reg_base, dir, num);
1806 if (err)
1807 goto exit;
1808
1809 err = otx2_mbox_init(&mw->mbox_up, hwbase, rvu->pdev,
1810 reg_base, dir_up, num);
1811 if (err)
1812 goto exit;
1813
1814 for (i = 0; i < num; i++) {
1815 mwork = &mw->mbox_wrk[i];
1816 mwork->rvu = rvu;
1817 INIT_WORK(&mwork->work, mbox_handler);
1818
1819 mwork = &mw->mbox_wrk_up[i];
1820 mwork->rvu = rvu;
1821 INIT_WORK(&mwork->work, mbox_up_handler);
1822 }
1823
1824 return 0;
1825 exit:
1826 if (hwbase)
1827 iounmap((void __iomem *)hwbase);
1828 destroy_workqueue(mw->mbox_wq);
1829 return err;
1830 }
1831
rvu_mbox_destroy(struct mbox_wq_info * mw)1832 static void rvu_mbox_destroy(struct mbox_wq_info *mw)
1833 {
1834 if (mw->mbox_wq) {
1835 flush_workqueue(mw->mbox_wq);
1836 destroy_workqueue(mw->mbox_wq);
1837 mw->mbox_wq = NULL;
1838 }
1839
1840 if (mw->mbox.hwbase)
1841 iounmap((void __iomem *)mw->mbox.hwbase);
1842
1843 otx2_mbox_destroy(&mw->mbox);
1844 otx2_mbox_destroy(&mw->mbox_up);
1845 }
1846
rvu_queue_work(struct mbox_wq_info * mw,int first,int mdevs,u64 intr)1847 static void rvu_queue_work(struct mbox_wq_info *mw, int first,
1848 int mdevs, u64 intr)
1849 {
1850 struct otx2_mbox_dev *mdev;
1851 struct otx2_mbox *mbox;
1852 struct mbox_hdr *hdr;
1853 int i;
1854
1855 for (i = first; i < mdevs; i++) {
1856 /* start from 0 */
1857 if (!(intr & BIT_ULL(i - first)))
1858 continue;
1859
1860 mbox = &mw->mbox;
1861 mdev = &mbox->dev[i];
1862 hdr = mdev->mbase + mbox->rx_start;
1863
1864 /*The hdr->num_msgs is set to zero immediately in the interrupt
1865 * handler to ensure that it holds a correct value next time
1866 * when the interrupt handler is called.
1867 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
1868 * pf>mbox.up_num_msgs holds the data for use in
1869 * pfaf_mbox_up_handler.
1870 */
1871
1872 if (hdr->num_msgs) {
1873 mw->mbox_wrk[i].num_msgs = hdr->num_msgs;
1874 hdr->num_msgs = 0;
1875 queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);
1876 }
1877 mbox = &mw->mbox_up;
1878 mdev = &mbox->dev[i];
1879 hdr = mdev->mbase + mbox->rx_start;
1880 if (hdr->num_msgs) {
1881 mw->mbox_wrk_up[i].up_num_msgs = hdr->num_msgs;
1882 hdr->num_msgs = 0;
1883 queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);
1884 }
1885 }
1886 }
1887
rvu_mbox_intr_handler(int irq,void * rvu_irq)1888 static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
1889 {
1890 struct rvu *rvu = (struct rvu *)rvu_irq;
1891 int vfs = rvu->vfs;
1892 u64 intr;
1893
1894 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
1895 /* Clear interrupts */
1896 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
1897 if (intr)
1898 trace_otx2_msg_interrupt(rvu->pdev, "PF(s) to AF", intr);
1899
1900 /* Sync with mbox memory region */
1901 rmb();
1902
1903 rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
1904
1905 /* Handle VF interrupts */
1906 if (vfs > 64) {
1907 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
1908 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);
1909
1910 rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);
1911 vfs -= 64;
1912 }
1913
1914 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
1915 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
1916 if (intr)
1917 trace_otx2_msg_interrupt(rvu->pdev, "VF(s) to AF", intr);
1918
1919 rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
1920
1921 return IRQ_HANDLED;
1922 }
1923
rvu_enable_mbox_intr(struct rvu * rvu)1924 static void rvu_enable_mbox_intr(struct rvu *rvu)
1925 {
1926 struct rvu_hwinfo *hw = rvu->hw;
1927
1928 /* Clear spurious irqs, if any */
1929 rvu_write64(rvu, BLKADDR_RVUM,
1930 RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));
1931
1932 /* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */
1933 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,
1934 INTR_MASK(hw->total_pfs) & ~1ULL);
1935 }
1936
rvu_blklf_teardown(struct rvu * rvu,u16 pcifunc,u8 blkaddr)1937 static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
1938 {
1939 struct rvu_block *block;
1940 int slot, lf, num_lfs;
1941 int err;
1942
1943 block = &rvu->hw->block[blkaddr];
1944 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1945 block->type);
1946 if (!num_lfs)
1947 return;
1948 for (slot = 0; slot < num_lfs; slot++) {
1949 lf = rvu_get_lf(rvu, block, pcifunc, slot);
1950 if (lf < 0)
1951 continue;
1952
1953 /* Cleanup LF and reset it */
1954 if (block->addr == BLKADDR_NIX0)
1955 rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);
1956 else if (block->addr == BLKADDR_NPA)
1957 rvu_npa_lf_teardown(rvu, pcifunc, lf);
1958
1959 err = rvu_lf_reset(rvu, block, lf);
1960 if (err) {
1961 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
1962 block->addr, lf);
1963 }
1964 }
1965 }
1966
__rvu_flr_handler(struct rvu * rvu,u16 pcifunc)1967 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)
1968 {
1969 mutex_lock(&rvu->flr_lock);
1970 /* Reset order should reflect inter-block dependencies:
1971 * 1. Reset any packet/work sources (NIX, CPT, TIM)
1972 * 2. Flush and reset SSO/SSOW
1973 * 3. Cleanup pools (NPA)
1974 */
1975 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);
1976 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);
1977 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);
1978 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);
1979 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);
1980 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);
1981 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1982 mutex_unlock(&rvu->flr_lock);
1983 }
1984
rvu_afvf_flr_handler(struct rvu * rvu,int vf)1985 static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)
1986 {
1987 int reg = 0;
1988
1989 /* pcifunc = 0(PF0) | (vf + 1) */
1990 __rvu_flr_handler(rvu, vf + 1);
1991
1992 if (vf >= 64) {
1993 reg = 1;
1994 vf = vf - 64;
1995 }
1996
1997 /* Signal FLR finish and enable IRQ */
1998 rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
1999 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
2000 }
2001
rvu_flr_handler(struct work_struct * work)2002 static void rvu_flr_handler(struct work_struct *work)
2003 {
2004 struct rvu_work *flrwork = container_of(work, struct rvu_work, work);
2005 struct rvu *rvu = flrwork->rvu;
2006 u16 pcifunc, numvfs, vf;
2007 u64 cfg;
2008 int pf;
2009
2010 pf = flrwork - rvu->flr_wrk;
2011 if (pf >= rvu->hw->total_pfs) {
2012 rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);
2013 return;
2014 }
2015
2016 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2017 numvfs = (cfg >> 12) & 0xFF;
2018 pcifunc = pf << RVU_PFVF_PF_SHIFT;
2019
2020 for (vf = 0; vf < numvfs; vf++)
2021 __rvu_flr_handler(rvu, (pcifunc | (vf + 1)));
2022
2023 __rvu_flr_handler(rvu, pcifunc);
2024
2025 /* Signal FLR finish */
2026 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));
2027
2028 /* Enable interrupt */
2029 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S, BIT_ULL(pf));
2030 }
2031
rvu_afvf_queue_flr_work(struct rvu * rvu,int start_vf,int numvfs)2032 static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)
2033 {
2034 int dev, vf, reg = 0;
2035 u64 intr;
2036
2037 if (start_vf >= 64)
2038 reg = 1;
2039
2040 intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));
2041 if (!intr)
2042 return;
2043
2044 for (vf = 0; vf < numvfs; vf++) {
2045 if (!(intr & BIT_ULL(vf)))
2046 continue;
2047 dev = vf + start_vf + rvu->hw->total_pfs;
2048 queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);
2049 /* Clear and disable the interrupt */
2050 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
2051 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));
2052 }
2053 }
2054
rvu_flr_intr_handler(int irq,void * rvu_irq)2055 static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)
2056 {
2057 struct rvu *rvu = (struct rvu *)rvu_irq;
2058 u64 intr;
2059 u8 pf;
2060
2061 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);
2062 if (!intr)
2063 goto afvf_flr;
2064
2065 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2066 if (intr & (1ULL << pf)) {
2067 /* PF is already dead do only AF related operations */
2068 queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);
2069 /* clear interrupt */
2070 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,
2071 BIT_ULL(pf));
2072 /* Disable the interrupt */
2073 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
2074 BIT_ULL(pf));
2075 }
2076 }
2077
2078 afvf_flr:
2079 rvu_afvf_queue_flr_work(rvu, 0, 64);
2080 if (rvu->vfs > 64)
2081 rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);
2082
2083 return IRQ_HANDLED;
2084 }
2085
rvu_me_handle_vfset(struct rvu * rvu,int idx,u64 intr)2086 static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)
2087 {
2088 int vf;
2089
2090 /* Nothing to be done here other than clearing the
2091 * TRPEND bit.
2092 */
2093 for (vf = 0; vf < 64; vf++) {
2094 if (intr & (1ULL << vf)) {
2095 /* clear the trpend due to ME(master enable) */
2096 rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));
2097 /* clear interrupt */
2098 rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));
2099 }
2100 }
2101 }
2102
2103 /* Handles ME interrupts from VFs of AF */
rvu_me_vf_intr_handler(int irq,void * rvu_irq)2104 static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)
2105 {
2106 struct rvu *rvu = (struct rvu *)rvu_irq;
2107 int vfset;
2108 u64 intr;
2109
2110 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
2111
2112 for (vfset = 0; vfset <= 1; vfset++) {
2113 intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));
2114 if (intr)
2115 rvu_me_handle_vfset(rvu, vfset, intr);
2116 }
2117
2118 return IRQ_HANDLED;
2119 }
2120
2121 /* Handles ME interrupts from PFs */
rvu_me_pf_intr_handler(int irq,void * rvu_irq)2122 static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)
2123 {
2124 struct rvu *rvu = (struct rvu *)rvu_irq;
2125 u64 intr;
2126 u8 pf;
2127
2128 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
2129
2130 /* Nothing to be done here other than clearing the
2131 * TRPEND bit.
2132 */
2133 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2134 if (intr & (1ULL << pf)) {
2135 /* clear the trpend due to ME(master enable) */
2136 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,
2137 BIT_ULL(pf));
2138 /* clear interrupt */
2139 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,
2140 BIT_ULL(pf));
2141 }
2142 }
2143
2144 return IRQ_HANDLED;
2145 }
2146
rvu_unregister_interrupts(struct rvu * rvu)2147 static void rvu_unregister_interrupts(struct rvu *rvu)
2148 {
2149 int irq;
2150
2151 /* Disable the Mbox interrupt */
2152 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,
2153 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2154
2155 /* Disable the PF FLR interrupt */
2156 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
2157 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2158
2159 /* Disable the PF ME interrupt */
2160 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,
2161 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2162
2163 for (irq = 0; irq < rvu->num_vec; irq++) {
2164 if (rvu->irq_allocated[irq]) {
2165 free_irq(pci_irq_vector(rvu->pdev, irq), rvu);
2166 rvu->irq_allocated[irq] = false;
2167 }
2168 }
2169
2170 pci_free_irq_vectors(rvu->pdev);
2171 rvu->num_vec = 0;
2172 }
2173
rvu_afvf_msix_vectors_num_ok(struct rvu * rvu)2174 static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
2175 {
2176 struct rvu_pfvf *pfvf = &rvu->pf[0];
2177 int offset;
2178
2179 pfvf = &rvu->pf[0];
2180 offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2181
2182 /* Make sure there are enough MSIX vectors configured so that
2183 * VF interrupts can be handled. Offset equal to zero means
2184 * that PF vectors are not configured and overlapping AF vectors.
2185 */
2186 return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&
2187 offset;
2188 }
2189
rvu_register_interrupts(struct rvu * rvu)2190 static int rvu_register_interrupts(struct rvu *rvu)
2191 {
2192 int ret, offset, pf_vec_start;
2193
2194 rvu->num_vec = pci_msix_vec_count(rvu->pdev);
2195
2196 rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
2197 NAME_SIZE, GFP_KERNEL);
2198 if (!rvu->irq_name)
2199 return -ENOMEM;
2200
2201 rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,
2202 sizeof(bool), GFP_KERNEL);
2203 if (!rvu->irq_allocated)
2204 return -ENOMEM;
2205
2206 /* Enable MSI-X */
2207 ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,
2208 rvu->num_vec, PCI_IRQ_MSIX);
2209 if (ret < 0) {
2210 dev_err(rvu->dev,
2211 "RVUAF: Request for %d msix vectors failed, ret %d\n",
2212 rvu->num_vec, ret);
2213 return ret;
2214 }
2215
2216 /* Register mailbox interrupt handler */
2217 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");
2218 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),
2219 rvu_mbox_intr_handler, 0,
2220 &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);
2221 if (ret) {
2222 dev_err(rvu->dev,
2223 "RVUAF: IRQ registration failed for mbox irq\n");
2224 goto fail;
2225 }
2226
2227 rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;
2228
2229 /* Enable mailbox interrupts from all PFs */
2230 rvu_enable_mbox_intr(rvu);
2231
2232 /* Register FLR interrupt handler */
2233 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2234 "RVUAF FLR");
2235 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),
2236 rvu_flr_intr_handler, 0,
2237 &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2238 rvu);
2239 if (ret) {
2240 dev_err(rvu->dev,
2241 "RVUAF: IRQ registration failed for FLR\n");
2242 goto fail;
2243 }
2244 rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;
2245
2246 /* Enable FLR interrupt for all PFs*/
2247 rvu_write64(rvu, BLKADDR_RVUM,
2248 RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));
2249
2250 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,
2251 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2252
2253 /* Register ME interrupt handler */
2254 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2255 "RVUAF ME");
2256 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),
2257 rvu_me_pf_intr_handler, 0,
2258 &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2259 rvu);
2260 if (ret) {
2261 dev_err(rvu->dev,
2262 "RVUAF: IRQ registration failed for ME\n");
2263 }
2264 rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;
2265
2266 /* Clear TRPEND bit for all PF */
2267 rvu_write64(rvu, BLKADDR_RVUM,
2268 RVU_AF_PFTRPEND, INTR_MASK(rvu->hw->total_pfs));
2269 /* Enable ME interrupt for all PFs*/
2270 rvu_write64(rvu, BLKADDR_RVUM,
2271 RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));
2272
2273 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,
2274 INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2275
2276 if (!rvu_afvf_msix_vectors_num_ok(rvu))
2277 return 0;
2278
2279 /* Get PF MSIX vectors offset. */
2280 pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,
2281 RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2282
2283 /* Register MBOX0 interrupt. */
2284 offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;
2285 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");
2286 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2287 rvu_mbox_intr_handler, 0,
2288 &rvu->irq_name[offset * NAME_SIZE],
2289 rvu);
2290 if (ret)
2291 dev_err(rvu->dev,
2292 "RVUAF: IRQ registration failed for Mbox0\n");
2293
2294 rvu->irq_allocated[offset] = true;
2295
2296 /* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so
2297 * simply increment current offset by 1.
2298 */
2299 offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;
2300 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");
2301 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2302 rvu_mbox_intr_handler, 0,
2303 &rvu->irq_name[offset * NAME_SIZE],
2304 rvu);
2305 if (ret)
2306 dev_err(rvu->dev,
2307 "RVUAF: IRQ registration failed for Mbox1\n");
2308
2309 rvu->irq_allocated[offset] = true;
2310
2311 /* Register FLR interrupt handler for AF's VFs */
2312 offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;
2313 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");
2314 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2315 rvu_flr_intr_handler, 0,
2316 &rvu->irq_name[offset * NAME_SIZE], rvu);
2317 if (ret) {
2318 dev_err(rvu->dev,
2319 "RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
2320 goto fail;
2321 }
2322 rvu->irq_allocated[offset] = true;
2323
2324 offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;
2325 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");
2326 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2327 rvu_flr_intr_handler, 0,
2328 &rvu->irq_name[offset * NAME_SIZE], rvu);
2329 if (ret) {
2330 dev_err(rvu->dev,
2331 "RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
2332 goto fail;
2333 }
2334 rvu->irq_allocated[offset] = true;
2335
2336 /* Register ME interrupt handler for AF's VFs */
2337 offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;
2338 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");
2339 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2340 rvu_me_vf_intr_handler, 0,
2341 &rvu->irq_name[offset * NAME_SIZE], rvu);
2342 if (ret) {
2343 dev_err(rvu->dev,
2344 "RVUAF: IRQ registration failed for RVUAFVF ME0\n");
2345 goto fail;
2346 }
2347 rvu->irq_allocated[offset] = true;
2348
2349 offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;
2350 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");
2351 ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2352 rvu_me_vf_intr_handler, 0,
2353 &rvu->irq_name[offset * NAME_SIZE], rvu);
2354 if (ret) {
2355 dev_err(rvu->dev,
2356 "RVUAF: IRQ registration failed for RVUAFVF ME1\n");
2357 goto fail;
2358 }
2359 rvu->irq_allocated[offset] = true;
2360 return 0;
2361
2362 fail:
2363 rvu_unregister_interrupts(rvu);
2364 return ret;
2365 }
2366
rvu_flr_wq_destroy(struct rvu * rvu)2367 static void rvu_flr_wq_destroy(struct rvu *rvu)
2368 {
2369 if (rvu->flr_wq) {
2370 flush_workqueue(rvu->flr_wq);
2371 destroy_workqueue(rvu->flr_wq);
2372 rvu->flr_wq = NULL;
2373 }
2374 }
2375
rvu_flr_init(struct rvu * rvu)2376 static int rvu_flr_init(struct rvu *rvu)
2377 {
2378 int dev, num_devs;
2379 u64 cfg;
2380 int pf;
2381
2382 /* Enable FLR for all PFs*/
2383 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2384 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2385 rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),
2386 cfg | BIT_ULL(22));
2387 }
2388
2389 rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
2390 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
2391 1);
2392 if (!rvu->flr_wq)
2393 return -ENOMEM;
2394
2395 num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);
2396 rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,
2397 sizeof(struct rvu_work), GFP_KERNEL);
2398 if (!rvu->flr_wrk) {
2399 destroy_workqueue(rvu->flr_wq);
2400 return -ENOMEM;
2401 }
2402
2403 for (dev = 0; dev < num_devs; dev++) {
2404 rvu->flr_wrk[dev].rvu = rvu;
2405 INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);
2406 }
2407
2408 mutex_init(&rvu->flr_lock);
2409
2410 return 0;
2411 }
2412
rvu_disable_afvf_intr(struct rvu * rvu)2413 static void rvu_disable_afvf_intr(struct rvu *rvu)
2414 {
2415 int vfs = rvu->vfs;
2416
2417 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));
2418 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
2419 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
2420 if (vfs <= 64)
2421 return;
2422
2423 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
2424 INTR_MASK(vfs - 64));
2425 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2426 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2427 }
2428
rvu_enable_afvf_intr(struct rvu * rvu)2429 static void rvu_enable_afvf_intr(struct rvu *rvu)
2430 {
2431 int vfs = rvu->vfs;
2432
2433 /* Clear any pending interrupts and enable AF VF interrupts for
2434 * the first 64 VFs.
2435 */
2436 /* Mbox */
2437 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));
2438 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));
2439
2440 /* FLR */
2441 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));
2442 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));
2443 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));
2444
2445 /* Same for remaining VFs, if any. */
2446 if (vfs <= 64)
2447 return;
2448
2449 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));
2450 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
2451 INTR_MASK(vfs - 64));
2452
2453 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));
2454 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2455 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2456 }
2457
2458 #define PCI_DEVID_OCTEONTX2_LBK 0xA061
2459
lbk_get_num_chans(void)2460 static int lbk_get_num_chans(void)
2461 {
2462 struct pci_dev *pdev;
2463 void __iomem *base;
2464 int ret = -EIO;
2465
2466 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,
2467 NULL);
2468 if (!pdev)
2469 goto err;
2470
2471 base = pci_ioremap_bar(pdev, 0);
2472 if (!base)
2473 goto err_put;
2474
2475 /* Read number of available LBK channels from LBK(0)_CONST register. */
2476 ret = (readq(base + 0x10) >> 32) & 0xffff;
2477 iounmap(base);
2478 err_put:
2479 pci_dev_put(pdev);
2480 err:
2481 return ret;
2482 }
2483
rvu_enable_sriov(struct rvu * rvu)2484 static int rvu_enable_sriov(struct rvu *rvu)
2485 {
2486 struct pci_dev *pdev = rvu->pdev;
2487 int err, chans, vfs;
2488
2489 if (!rvu_afvf_msix_vectors_num_ok(rvu)) {
2490 dev_warn(&pdev->dev,
2491 "Skipping SRIOV enablement since not enough IRQs are available\n");
2492 return 0;
2493 }
2494
2495 chans = lbk_get_num_chans();
2496 if (chans < 0)
2497 return chans;
2498
2499 vfs = pci_sriov_get_totalvfs(pdev);
2500
2501 /* Limit VFs in case we have more VFs than LBK channels available. */
2502 if (vfs > chans)
2503 vfs = chans;
2504
2505 if (!vfs)
2506 return 0;
2507
2508 /* Save VFs number for reference in VF interrupts handlers.
2509 * Since interrupts might start arriving during SRIOV enablement
2510 * ordinary API cannot be used to get number of enabled VFs.
2511 */
2512 rvu->vfs = vfs;
2513
2514 err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,
2515 rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);
2516 if (err)
2517 return err;
2518
2519 rvu_enable_afvf_intr(rvu);
2520 /* Make sure IRQs are enabled before SRIOV. */
2521 mb();
2522
2523 err = pci_enable_sriov(pdev, vfs);
2524 if (err) {
2525 rvu_disable_afvf_intr(rvu);
2526 rvu_mbox_destroy(&rvu->afvf_wq_info);
2527 return err;
2528 }
2529
2530 return 0;
2531 }
2532
rvu_disable_sriov(struct rvu * rvu)2533 static void rvu_disable_sriov(struct rvu *rvu)
2534 {
2535 rvu_disable_afvf_intr(rvu);
2536 rvu_mbox_destroy(&rvu->afvf_wq_info);
2537 pci_disable_sriov(rvu->pdev);
2538 }
2539
rvu_update_module_params(struct rvu * rvu)2540 static void rvu_update_module_params(struct rvu *rvu)
2541 {
2542 const char *default_pfl_name = "default";
2543
2544 strscpy(rvu->mkex_pfl_name,
2545 mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);
2546 }
2547
rvu_probe(struct pci_dev * pdev,const struct pci_device_id * id)2548 static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2549 {
2550 struct device *dev = &pdev->dev;
2551 struct rvu *rvu;
2552 int err;
2553
2554 rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);
2555 if (!rvu)
2556 return -ENOMEM;
2557
2558 rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);
2559 if (!rvu->hw) {
2560 devm_kfree(dev, rvu);
2561 return -ENOMEM;
2562 }
2563
2564 pci_set_drvdata(pdev, rvu);
2565 rvu->pdev = pdev;
2566 rvu->dev = &pdev->dev;
2567
2568 err = pci_enable_device(pdev);
2569 if (err) {
2570 dev_err(dev, "Failed to enable PCI device\n");
2571 goto err_freemem;
2572 }
2573
2574 err = pci_request_regions(pdev, DRV_NAME);
2575 if (err) {
2576 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2577 goto err_disable_device;
2578 }
2579
2580 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
2581 if (err) {
2582 dev_err(dev, "DMA mask config failed, abort\n");
2583 goto err_release_regions;
2584 }
2585
2586 pci_set_master(pdev);
2587
2588 rvu->ptp = ptp_get();
2589 if (IS_ERR(rvu->ptp)) {
2590 err = PTR_ERR(rvu->ptp);
2591 if (err == -EPROBE_DEFER)
2592 goto err_release_regions;
2593 rvu->ptp = NULL;
2594 }
2595
2596 /* Map Admin function CSRs */
2597 rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);
2598 rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
2599 if (!rvu->afreg_base || !rvu->pfreg_base) {
2600 dev_err(dev, "Unable to map admin function CSRs, aborting\n");
2601 err = -ENOMEM;
2602 goto err_put_ptp;
2603 }
2604
2605 /* Store module params in rvu structure */
2606 rvu_update_module_params(rvu);
2607
2608 /* Check which blocks the HW supports */
2609 rvu_check_block_implemented(rvu);
2610
2611 rvu_reset_all_blocks(rvu);
2612
2613 rvu_setup_hw_capabilities(rvu);
2614
2615 err = rvu_setup_hw_resources(rvu);
2616 if (err)
2617 goto err_put_ptp;
2618
2619 /* Init mailbox btw AF and PFs */
2620 err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,
2621 rvu->hw->total_pfs, rvu_afpf_mbox_handler,
2622 rvu_afpf_mbox_up_handler);
2623 if (err)
2624 goto err_hwsetup;
2625
2626 err = rvu_flr_init(rvu);
2627 if (err)
2628 goto err_mbox;
2629
2630 err = rvu_register_interrupts(rvu);
2631 if (err)
2632 goto err_flr;
2633
2634 rvu_setup_rvum_blk_revid(rvu);
2635
2636 /* Enable AF's VFs (if any) */
2637 err = rvu_enable_sriov(rvu);
2638 if (err)
2639 goto err_irq;
2640
2641 /* Initialize debugfs */
2642 rvu_dbg_init(rvu);
2643
2644 return 0;
2645 err_irq:
2646 rvu_unregister_interrupts(rvu);
2647 err_flr:
2648 rvu_flr_wq_destroy(rvu);
2649 err_mbox:
2650 rvu_mbox_destroy(&rvu->afpf_wq_info);
2651 err_hwsetup:
2652 rvu_cgx_exit(rvu);
2653 rvu_fwdata_exit(rvu);
2654 rvu_reset_all_blocks(rvu);
2655 rvu_free_hw_resources(rvu);
2656 rvu_clear_rvum_blk_revid(rvu);
2657 err_put_ptp:
2658 ptp_put(rvu->ptp);
2659 err_release_regions:
2660 pci_release_regions(pdev);
2661 err_disable_device:
2662 pci_disable_device(pdev);
2663 err_freemem:
2664 pci_set_drvdata(pdev, NULL);
2665 devm_kfree(&pdev->dev, rvu->hw);
2666 devm_kfree(dev, rvu);
2667 return err;
2668 }
2669
rvu_remove(struct pci_dev * pdev)2670 static void rvu_remove(struct pci_dev *pdev)
2671 {
2672 struct rvu *rvu = pci_get_drvdata(pdev);
2673
2674 rvu_dbg_exit(rvu);
2675 rvu_unregister_interrupts(rvu);
2676 rvu_flr_wq_destroy(rvu);
2677 rvu_cgx_exit(rvu);
2678 rvu_fwdata_exit(rvu);
2679 rvu_mbox_destroy(&rvu->afpf_wq_info);
2680 rvu_disable_sriov(rvu);
2681 rvu_reset_all_blocks(rvu);
2682 rvu_free_hw_resources(rvu);
2683 rvu_clear_rvum_blk_revid(rvu);
2684 ptp_put(rvu->ptp);
2685 pci_release_regions(pdev);
2686 pci_disable_device(pdev);
2687 pci_set_drvdata(pdev, NULL);
2688
2689 devm_kfree(&pdev->dev, rvu->hw);
2690 devm_kfree(&pdev->dev, rvu);
2691 }
2692
2693 static struct pci_driver rvu_driver = {
2694 .name = DRV_NAME,
2695 .id_table = rvu_id_table,
2696 .probe = rvu_probe,
2697 .remove = rvu_remove,
2698 };
2699
rvu_init_module(void)2700 static int __init rvu_init_module(void)
2701 {
2702 int err;
2703
2704 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
2705
2706 err = pci_register_driver(&cgx_driver);
2707 if (err < 0)
2708 return err;
2709
2710 err = pci_register_driver(&ptp_driver);
2711 if (err < 0)
2712 goto ptp_err;
2713
2714 err = pci_register_driver(&rvu_driver);
2715 if (err < 0)
2716 goto rvu_err;
2717
2718 return 0;
2719 rvu_err:
2720 pci_unregister_driver(&ptp_driver);
2721 ptp_err:
2722 pci_unregister_driver(&cgx_driver);
2723
2724 return err;
2725 }
2726
rvu_cleanup_module(void)2727 static void __exit rvu_cleanup_module(void)
2728 {
2729 pci_unregister_driver(&rvu_driver);
2730 pci_unregister_driver(&ptp_driver);
2731 pci_unregister_driver(&cgx_driver);
2732 }
2733
2734 module_init(rvu_init_module);
2735 module_exit(rvu_cleanup_module);
2736