1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell OcteonTx2 RVU Admin Function driver
3 *
4 * Copyright (C) 2018 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12 #include <linux/pci.h>
13
14 #include "rvu_struct.h"
15 #include "rvu_reg.h"
16 #include "rvu.h"
17
npa_aq_enqueue_wait(struct rvu * rvu,struct rvu_block * block,struct npa_aq_inst_s * inst)18 static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
19 struct npa_aq_inst_s *inst)
20 {
21 struct admin_queue *aq = block->aq;
22 struct npa_aq_res_s *result;
23 int timeout = 1000;
24 u64 reg, head;
25
26 result = (struct npa_aq_res_s *)aq->res->base;
27
28 /* Get current head pointer where to append this instruction */
29 reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS);
30 head = (reg >> 4) & AQ_PTR_MASK;
31
32 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
33 (void *)inst, aq->inst->entry_sz);
34 memset(result, 0, sizeof(*result));
35 /* sync into memory */
36 wmb();
37
38 /* Ring the doorbell and wait for result */
39 rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1);
40 while (result->compcode == NPA_AQ_COMP_NOTDONE) {
41 cpu_relax();
42 udelay(1);
43 timeout--;
44 if (!timeout)
45 return -EBUSY;
46 }
47
48 if (result->compcode != NPA_AQ_COMP_GOOD)
49 /* TODO: Replace this with some error code */
50 return -EBUSY;
51
52 return 0;
53 }
54
rvu_npa_aq_enq_inst(struct rvu * rvu,struct npa_aq_enq_req * req,struct npa_aq_enq_rsp * rsp)55 static int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
56 struct npa_aq_enq_rsp *rsp)
57 {
58 struct rvu_hwinfo *hw = rvu->hw;
59 u16 pcifunc = req->hdr.pcifunc;
60 int blkaddr, npalf, rc = 0;
61 struct npa_aq_inst_s inst;
62 struct rvu_block *block;
63 struct admin_queue *aq;
64 struct rvu_pfvf *pfvf;
65 void *ctx, *mask;
66 bool ena;
67
68 pfvf = rvu_get_pfvf(rvu, pcifunc);
69 if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize)
70 return NPA_AF_ERR_AQ_ENQUEUE;
71
72 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
73 if (!pfvf->npalf || blkaddr < 0)
74 return NPA_AF_ERR_AF_LF_INVALID;
75
76 block = &hw->block[blkaddr];
77 aq = block->aq;
78 if (!aq) {
79 dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__);
80 return NPA_AF_ERR_AQ_ENQUEUE;
81 }
82
83 npalf = rvu_get_lf(rvu, block, pcifunc, 0);
84 if (npalf < 0)
85 return NPA_AF_ERR_AF_LF_INVALID;
86
87 memset(&inst, 0, sizeof(struct npa_aq_inst_s));
88 inst.cindex = req->aura_id;
89 inst.lf = npalf;
90 inst.ctype = req->ctype;
91 inst.op = req->op;
92 /* Currently we are not supporting enqueuing multiple instructions,
93 * so always choose first entry in result memory.
94 */
95 inst.res_addr = (u64)aq->res->iova;
96
97 /* Clean result + context memory */
98 memset(aq->res->base, 0, aq->res->entry_sz);
99 /* Context needs to be written at RES_ADDR + 128 */
100 ctx = aq->res->base + 128;
101 /* Mask needs to be written at RES_ADDR + 256 */
102 mask = aq->res->base + 256;
103
104 switch (req->op) {
105 case NPA_AQ_INSTOP_WRITE:
106 /* Copy context and write mask */
107 if (req->ctype == NPA_AQ_CTYPE_AURA) {
108 memcpy(mask, &req->aura_mask,
109 sizeof(struct npa_aura_s));
110 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
111 } else {
112 memcpy(mask, &req->pool_mask,
113 sizeof(struct npa_pool_s));
114 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
115 }
116 break;
117 case NPA_AQ_INSTOP_INIT:
118 if (req->ctype == NPA_AQ_CTYPE_AURA) {
119 if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) {
120 rc = NPA_AF_ERR_AQ_FULL;
121 break;
122 }
123 /* Set pool's context address */
124 req->aura.pool_addr = pfvf->pool_ctx->iova +
125 (req->aura.pool_addr * pfvf->pool_ctx->entry_sz);
126 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
127 } else { /* POOL's context */
128 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
129 }
130 break;
131 case NPA_AQ_INSTOP_NOP:
132 case NPA_AQ_INSTOP_READ:
133 case NPA_AQ_INSTOP_LOCK:
134 case NPA_AQ_INSTOP_UNLOCK:
135 break;
136 default:
137 rc = NPA_AF_ERR_AQ_FULL;
138 break;
139 }
140
141 if (rc)
142 return rc;
143
144 spin_lock(&aq->lock);
145
146 /* Submit the instruction to AQ */
147 rc = npa_aq_enqueue_wait(rvu, block, &inst);
148 if (rc) {
149 spin_unlock(&aq->lock);
150 return rc;
151 }
152
153 /* Set aura bitmap if aura hw context is enabled */
154 if (req->ctype == NPA_AQ_CTYPE_AURA) {
155 if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena)
156 __set_bit(req->aura_id, pfvf->aura_bmap);
157 if (req->op == NPA_AQ_INSTOP_WRITE) {
158 ena = (req->aura.ena & req->aura_mask.ena) |
159 (test_bit(req->aura_id, pfvf->aura_bmap) &
160 ~req->aura_mask.ena);
161 if (ena)
162 __set_bit(req->aura_id, pfvf->aura_bmap);
163 else
164 __clear_bit(req->aura_id, pfvf->aura_bmap);
165 }
166 }
167
168 /* Set pool bitmap if pool hw context is enabled */
169 if (req->ctype == NPA_AQ_CTYPE_POOL) {
170 if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena)
171 __set_bit(req->aura_id, pfvf->pool_bmap);
172 if (req->op == NPA_AQ_INSTOP_WRITE) {
173 ena = (req->pool.ena & req->pool_mask.ena) |
174 (test_bit(req->aura_id, pfvf->pool_bmap) &
175 ~req->pool_mask.ena);
176 if (ena)
177 __set_bit(req->aura_id, pfvf->pool_bmap);
178 else
179 __clear_bit(req->aura_id, pfvf->pool_bmap);
180 }
181 }
182 spin_unlock(&aq->lock);
183
184 if (rsp) {
185 /* Copy read context into mailbox */
186 if (req->op == NPA_AQ_INSTOP_READ) {
187 if (req->ctype == NPA_AQ_CTYPE_AURA)
188 memcpy(&rsp->aura, ctx,
189 sizeof(struct npa_aura_s));
190 else
191 memcpy(&rsp->pool, ctx,
192 sizeof(struct npa_pool_s));
193 }
194 }
195
196 return 0;
197 }
198
npa_lf_hwctx_disable(struct rvu * rvu,struct hwctx_disable_req * req)199 static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
200 {
201 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
202 struct npa_aq_enq_req aq_req;
203 unsigned long *bmap;
204 int id, cnt = 0;
205 int err = 0, rc;
206
207 if (!pfvf->pool_ctx || !pfvf->aura_ctx)
208 return NPA_AF_ERR_AQ_ENQUEUE;
209
210 memset(&aq_req, 0, sizeof(struct npa_aq_enq_req));
211 aq_req.hdr.pcifunc = req->hdr.pcifunc;
212
213 if (req->ctype == NPA_AQ_CTYPE_POOL) {
214 aq_req.pool.ena = 0;
215 aq_req.pool_mask.ena = 1;
216 cnt = pfvf->pool_ctx->qsize;
217 bmap = pfvf->pool_bmap;
218 } else if (req->ctype == NPA_AQ_CTYPE_AURA) {
219 aq_req.aura.ena = 0;
220 aq_req.aura_mask.ena = 1;
221 cnt = pfvf->aura_ctx->qsize;
222 bmap = pfvf->aura_bmap;
223 }
224
225 aq_req.ctype = req->ctype;
226 aq_req.op = NPA_AQ_INSTOP_WRITE;
227
228 for (id = 0; id < cnt; id++) {
229 if (!test_bit(id, bmap))
230 continue;
231 aq_req.aura_id = id;
232 rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL);
233 if (rc) {
234 err = rc;
235 dev_err(rvu->dev, "Failed to disable %s:%d context\n",
236 (req->ctype == NPA_AQ_CTYPE_AURA) ?
237 "Aura" : "Pool", id);
238 }
239 }
240
241 return err;
242 }
243
rvu_mbox_handler_npa_aq_enq(struct rvu * rvu,struct npa_aq_enq_req * req,struct npa_aq_enq_rsp * rsp)244 int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
245 struct npa_aq_enq_req *req,
246 struct npa_aq_enq_rsp *rsp)
247 {
248 return rvu_npa_aq_enq_inst(rvu, req, rsp);
249 }
250
rvu_mbox_handler_npa_hwctx_disable(struct rvu * rvu,struct hwctx_disable_req * req,struct msg_rsp * rsp)251 int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu,
252 struct hwctx_disable_req *req,
253 struct msg_rsp *rsp)
254 {
255 return npa_lf_hwctx_disable(rvu, req);
256 }
257
npa_ctx_free(struct rvu * rvu,struct rvu_pfvf * pfvf)258 static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
259 {
260 kfree(pfvf->aura_bmap);
261 pfvf->aura_bmap = NULL;
262
263 qmem_free(rvu->dev, pfvf->aura_ctx);
264 pfvf->aura_ctx = NULL;
265
266 kfree(pfvf->pool_bmap);
267 pfvf->pool_bmap = NULL;
268
269 qmem_free(rvu->dev, pfvf->pool_ctx);
270 pfvf->pool_ctx = NULL;
271
272 qmem_free(rvu->dev, pfvf->npa_qints_ctx);
273 pfvf->npa_qints_ctx = NULL;
274 }
275
rvu_mbox_handler_npa_lf_alloc(struct rvu * rvu,struct npa_lf_alloc_req * req,struct npa_lf_alloc_rsp * rsp)276 int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu,
277 struct npa_lf_alloc_req *req,
278 struct npa_lf_alloc_rsp *rsp)
279 {
280 int npalf, qints, hwctx_size, err, rc = 0;
281 struct rvu_hwinfo *hw = rvu->hw;
282 u16 pcifunc = req->hdr.pcifunc;
283 struct rvu_block *block;
284 struct rvu_pfvf *pfvf;
285 u64 cfg, ctx_cfg;
286 int blkaddr;
287
288 if (req->aura_sz > NPA_AURA_SZ_MAX ||
289 req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools)
290 return NPA_AF_ERR_PARAM;
291
292 pfvf = rvu_get_pfvf(rvu, pcifunc);
293 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
294 if (!pfvf->npalf || blkaddr < 0)
295 return NPA_AF_ERR_AF_LF_INVALID;
296
297 block = &hw->block[blkaddr];
298 npalf = rvu_get_lf(rvu, block, pcifunc, 0);
299 if (npalf < 0)
300 return NPA_AF_ERR_AF_LF_INVALID;
301
302 /* Reset this NPA LF */
303 err = rvu_lf_reset(rvu, block, npalf);
304 if (err) {
305 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
306 return NPA_AF_ERR_LF_RESET;
307 }
308
309 ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1);
310
311 /* Alloc memory for aura HW contexts */
312 hwctx_size = 1UL << (ctx_cfg & 0xF);
313 err = qmem_alloc(rvu->dev, &pfvf->aura_ctx,
314 NPA_AURA_COUNT(req->aura_sz), hwctx_size);
315 if (err)
316 goto free_mem;
317
318 pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
319 GFP_KERNEL);
320 if (!pfvf->aura_bmap)
321 goto free_mem;
322
323 /* Alloc memory for pool HW contexts */
324 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
325 err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size);
326 if (err)
327 goto free_mem;
328
329 pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
330 GFP_KERNEL);
331 if (!pfvf->pool_bmap)
332 goto free_mem;
333
334 /* Get no of queue interrupts supported */
335 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
336 qints = (cfg >> 28) & 0xFFF;
337
338 /* Alloc memory for Qints HW contexts */
339 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
340 err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size);
341 if (err)
342 goto free_mem;
343
344 cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf));
345 /* Clear way partition mask and set aura offset to '0' */
346 cfg &= ~(BIT_ULL(34) - 1);
347 /* Set aura size & enable caching of contexts */
348 cfg |= (req->aura_sz << 16) | BIT_ULL(34);
349 rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg);
350
351 /* Configure aura HW context's base */
352 rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf),
353 (u64)pfvf->aura_ctx->iova);
354
355 /* Enable caching of qints hw context */
356 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf), BIT_ULL(36));
357 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf),
358 (u64)pfvf->npa_qints_ctx->iova);
359
360 goto exit;
361
362 free_mem:
363 npa_ctx_free(rvu, pfvf);
364 rc = -ENOMEM;
365
366 exit:
367 /* set stack page info */
368 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
369 rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF;
370 rsp->stack_pg_bytes = cfg & 0xFF;
371 rsp->qints = (cfg >> 28) & 0xFFF;
372 return rc;
373 }
374
rvu_mbox_handler_npa_lf_free(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)375 int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req,
376 struct msg_rsp *rsp)
377 {
378 struct rvu_hwinfo *hw = rvu->hw;
379 u16 pcifunc = req->hdr.pcifunc;
380 struct rvu_block *block;
381 struct rvu_pfvf *pfvf;
382 int npalf, err;
383 int blkaddr;
384
385 pfvf = rvu_get_pfvf(rvu, pcifunc);
386 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
387 if (!pfvf->npalf || blkaddr < 0)
388 return NPA_AF_ERR_AF_LF_INVALID;
389
390 block = &hw->block[blkaddr];
391 npalf = rvu_get_lf(rvu, block, pcifunc, 0);
392 if (npalf < 0)
393 return NPA_AF_ERR_AF_LF_INVALID;
394
395 /* Reset this NPA LF */
396 err = rvu_lf_reset(rvu, block, npalf);
397 if (err) {
398 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
399 return NPA_AF_ERR_LF_RESET;
400 }
401
402 npa_ctx_free(rvu, pfvf);
403
404 return 0;
405 }
406
npa_aq_init(struct rvu * rvu,struct rvu_block * block)407 static int npa_aq_init(struct rvu *rvu, struct rvu_block *block)
408 {
409 u64 cfg;
410 int err;
411
412 /* Set admin queue endianness */
413 cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG);
414 #ifdef __BIG_ENDIAN
415 cfg |= BIT_ULL(1);
416 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
417 #else
418 cfg &= ~BIT_ULL(1);
419 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
420 #endif
421
422 /* Do not bypass NDC cache */
423 cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG);
424 cfg &= ~0x03DULL;
425 rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg);
426
427 /* Result structure can be followed by Aura/Pool context at
428 * RES + 128bytes and a write mask at RES + 256 bytes, depending on
429 * operation type. Alloc sufficient result memory for all operations.
430 */
431 err = rvu_aq_alloc(rvu, &block->aq,
432 Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s),
433 ALIGN(sizeof(struct npa_aq_res_s), 128) + 256);
434 if (err)
435 return err;
436
437 rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE);
438 rvu_write64(rvu, block->addr,
439 NPA_AF_AQ_BASE, (u64)block->aq->inst->iova);
440 return 0;
441 }
442
rvu_npa_init(struct rvu * rvu)443 int rvu_npa_init(struct rvu *rvu)
444 {
445 struct rvu_hwinfo *hw = rvu->hw;
446 int blkaddr, err;
447
448 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
449 if (blkaddr < 0)
450 return 0;
451
452 /* Initialize admin queue */
453 err = npa_aq_init(rvu, &hw->block[blkaddr]);
454 if (err)
455 return err;
456
457 return 0;
458 }
459
rvu_npa_freemem(struct rvu * rvu)460 void rvu_npa_freemem(struct rvu *rvu)
461 {
462 struct rvu_hwinfo *hw = rvu->hw;
463 struct rvu_block *block;
464 int blkaddr;
465
466 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
467 if (blkaddr < 0)
468 return;
469
470 block = &hw->block[blkaddr];
471 rvu_aq_free(rvu, block->aq);
472 }
473
rvu_npa_lf_teardown(struct rvu * rvu,u16 pcifunc,int npalf)474 void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf)
475 {
476 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
477 struct hwctx_disable_req ctx_req;
478
479 /* Disable all pools */
480 ctx_req.hdr.pcifunc = pcifunc;
481 ctx_req.ctype = NPA_AQ_CTYPE_POOL;
482 npa_lf_hwctx_disable(rvu, &ctx_req);
483
484 /* Disable all auras */
485 ctx_req.ctype = NPA_AQ_CTYPE_AURA;
486 npa_lf_hwctx_disable(rvu, &ctx_req);
487
488 npa_ctx_free(rvu, pfvf);
489 }
490