1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell OcteonTX CPT driver
3 *
4 * Copyright (C) 2019 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include "otx_cptvf.h"
12 #include "otx_cptvf_algs.h"
13
14 /* Completion code size and initial value */
15 #define COMPLETION_CODE_SIZE 8
16 #define COMPLETION_CODE_INIT 0
17
18 /* SG list header size in bytes */
19 #define SG_LIST_HDR_SIZE 8
20
21 /* Default timeout when waiting for free pending entry in us */
22 #define CPT_PENTRY_TIMEOUT 1000
23 #define CPT_PENTRY_STEP 50
24
25 /* Default threshold for stopping and resuming sender requests */
26 #define CPT_IQ_STOP_MARGIN 128
27 #define CPT_IQ_RESUME_MARGIN 512
28
29 #define CPT_DMA_ALIGN 128
30
otx_cpt_dump_sg_list(struct pci_dev * pdev,struct otx_cpt_req_info * req)31 void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req)
32 {
33 int i;
34
35 pr_debug("Gather list size %d\n", req->incnt);
36 for (i = 0; i < req->incnt; i++) {
37 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
38 req->in[i].size, req->in[i].vptr,
39 (void *) req->in[i].dma_addr);
40 pr_debug("Buffer hexdump (%d bytes)\n",
41 req->in[i].size);
42 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
43 req->in[i].vptr, req->in[i].size, false);
44 }
45
46 pr_debug("Scatter list size %d\n", req->outcnt);
47 for (i = 0; i < req->outcnt; i++) {
48 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
49 req->out[i].size, req->out[i].vptr,
50 (void *) req->out[i].dma_addr);
51 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
52 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
53 req->out[i].vptr, req->out[i].size, false);
54 }
55 }
56
get_free_pending_entry(struct otx_cpt_pending_queue * q,int qlen)57 static inline struct otx_cpt_pending_entry *get_free_pending_entry(
58 struct otx_cpt_pending_queue *q,
59 int qlen)
60 {
61 struct otx_cpt_pending_entry *ent = NULL;
62
63 ent = &q->head[q->rear];
64 if (unlikely(ent->busy))
65 return NULL;
66
67 q->rear++;
68 if (unlikely(q->rear == qlen))
69 q->rear = 0;
70
71 return ent;
72 }
73
modulo_inc(u32 index,u32 length,u32 inc)74 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
75 {
76 if (WARN_ON(inc > length))
77 inc = length;
78
79 index += inc;
80 if (unlikely(index >= length))
81 index -= length;
82
83 return index;
84 }
85
free_pentry(struct otx_cpt_pending_entry * pentry)86 static inline void free_pentry(struct otx_cpt_pending_entry *pentry)
87 {
88 pentry->completion_addr = NULL;
89 pentry->info = NULL;
90 pentry->callback = NULL;
91 pentry->areq = NULL;
92 pentry->resume_sender = false;
93 pentry->busy = false;
94 }
95
setup_sgio_components(struct pci_dev * pdev,struct otx_cpt_buf_ptr * list,int buf_count,u8 * buffer)96 static inline int setup_sgio_components(struct pci_dev *pdev,
97 struct otx_cpt_buf_ptr *list,
98 int buf_count, u8 *buffer)
99 {
100 struct otx_cpt_sglist_component *sg_ptr = NULL;
101 int ret = 0, i, j;
102 int components;
103
104 if (unlikely(!list)) {
105 dev_err(&pdev->dev, "Input list pointer is NULL\n");
106 return -EFAULT;
107 }
108
109 for (i = 0; i < buf_count; i++) {
110 if (likely(list[i].vptr)) {
111 list[i].dma_addr = dma_map_single(&pdev->dev,
112 list[i].vptr,
113 list[i].size,
114 DMA_BIDIRECTIONAL);
115 if (unlikely(dma_mapping_error(&pdev->dev,
116 list[i].dma_addr))) {
117 dev_err(&pdev->dev, "Dma mapping failed\n");
118 ret = -EIO;
119 goto sg_cleanup;
120 }
121 }
122 }
123
124 components = buf_count / 4;
125 sg_ptr = (struct otx_cpt_sglist_component *)buffer;
126 for (i = 0; i < components; i++) {
127 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
128 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
129 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
130 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
131 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
132 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
133 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
134 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
135 sg_ptr++;
136 }
137 components = buf_count % 4;
138
139 switch (components) {
140 case 3:
141 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
142 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
143 fallthrough;
144 case 2:
145 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
146 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
147 fallthrough;
148 case 1:
149 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
150 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
151 break;
152 default:
153 break;
154 }
155 return ret;
156
157 sg_cleanup:
158 for (j = 0; j < i; j++) {
159 if (list[j].dma_addr) {
160 dma_unmap_single(&pdev->dev, list[i].dma_addr,
161 list[i].size, DMA_BIDIRECTIONAL);
162 }
163
164 list[j].dma_addr = 0;
165 }
166 return ret;
167 }
168
setup_sgio_list(struct pci_dev * pdev,struct otx_cpt_info_buffer ** pinfo,struct otx_cpt_req_info * req,gfp_t gfp)169 static inline int setup_sgio_list(struct pci_dev *pdev,
170 struct otx_cpt_info_buffer **pinfo,
171 struct otx_cpt_req_info *req, gfp_t gfp)
172 {
173 u32 dlen, align_dlen, info_len, rlen;
174 struct otx_cpt_info_buffer *info;
175 u16 g_sz_bytes, s_sz_bytes;
176 int align = CPT_DMA_ALIGN;
177 u32 total_mem_len;
178
179 if (unlikely(req->incnt > OTX_CPT_MAX_SG_IN_CNT ||
180 req->outcnt > OTX_CPT_MAX_SG_OUT_CNT)) {
181 dev_err(&pdev->dev, "Error too many sg components\n");
182 return -EINVAL;
183 }
184
185 g_sz_bytes = ((req->incnt + 3) / 4) *
186 sizeof(struct otx_cpt_sglist_component);
187 s_sz_bytes = ((req->outcnt + 3) / 4) *
188 sizeof(struct otx_cpt_sglist_component);
189
190 dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
191 align_dlen = ALIGN(dlen, align);
192 info_len = ALIGN(sizeof(*info), align);
193 rlen = ALIGN(sizeof(union otx_cpt_res_s), align);
194 total_mem_len = align_dlen + info_len + rlen + COMPLETION_CODE_SIZE;
195
196 info = kzalloc(total_mem_len, gfp);
197 if (unlikely(!info)) {
198 dev_err(&pdev->dev, "Memory allocation failed\n");
199 return -ENOMEM;
200 }
201 *pinfo = info;
202 info->dlen = dlen;
203 info->in_buffer = (u8 *)info + info_len;
204
205 ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
206 ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
207 ((u16 *)info->in_buffer)[2] = 0;
208 ((u16 *)info->in_buffer)[3] = 0;
209
210 /* Setup gather (input) components */
211 if (setup_sgio_components(pdev, req->in, req->incnt,
212 &info->in_buffer[8])) {
213 dev_err(&pdev->dev, "Failed to setup gather list\n");
214 return -EFAULT;
215 }
216
217 if (setup_sgio_components(pdev, req->out, req->outcnt,
218 &info->in_buffer[8 + g_sz_bytes])) {
219 dev_err(&pdev->dev, "Failed to setup scatter list\n");
220 return -EFAULT;
221 }
222
223 info->dma_len = total_mem_len - info_len;
224 info->dptr_baddr = dma_map_single(&pdev->dev, (void *)info->in_buffer,
225 info->dma_len, DMA_BIDIRECTIONAL);
226 if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
227 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
228 return -EIO;
229 }
230 /*
231 * Get buffer for union otx_cpt_res_s response
232 * structure and its physical address
233 */
234 info->completion_addr = (u64 *)(info->in_buffer + align_dlen);
235 info->comp_baddr = info->dptr_baddr + align_dlen;
236
237 /* Create and initialize RPTR */
238 info->out_buffer = (u8 *)info->completion_addr + rlen;
239 info->rptr_baddr = info->comp_baddr + rlen;
240
241 *((u64 *) info->out_buffer) = ~((u64) COMPLETION_CODE_INIT);
242
243 return 0;
244 }
245
246
cpt_fill_inst(union otx_cpt_inst_s * inst,struct otx_cpt_info_buffer * info,struct otx_cpt_iq_cmd * cmd)247 static void cpt_fill_inst(union otx_cpt_inst_s *inst,
248 struct otx_cpt_info_buffer *info,
249 struct otx_cpt_iq_cmd *cmd)
250 {
251 inst->u[0] = 0x0;
252 inst->s.doneint = true;
253 inst->s.res_addr = (u64)info->comp_baddr;
254 inst->u[2] = 0x0;
255 inst->s.wq_ptr = 0;
256 inst->s.ei0 = cmd->cmd.u64;
257 inst->s.ei1 = cmd->dptr;
258 inst->s.ei2 = cmd->rptr;
259 inst->s.ei3 = cmd->cptr.u64;
260 }
261
262 /*
263 * On OcteonTX platform the parameter db_count is used as a count for ringing
264 * door bell. The valid values for db_count are:
265 * 0 - 1 CPT instruction will be enqueued however CPT will not be informed
266 * 1 - 1 CPT instruction will be enqueued and CPT will be informed
267 */
cpt_send_cmd(union otx_cpt_inst_s * cptinst,struct otx_cptvf * cptvf)268 static void cpt_send_cmd(union otx_cpt_inst_s *cptinst, struct otx_cptvf *cptvf)
269 {
270 struct otx_cpt_cmd_qinfo *qinfo = &cptvf->cqinfo;
271 struct otx_cpt_cmd_queue *queue;
272 struct otx_cpt_cmd_chunk *curr;
273 u8 *ent;
274
275 queue = &qinfo->queue[0];
276 /*
277 * cpt_send_cmd is currently called only from critical section
278 * therefore no locking is required for accessing instruction queue
279 */
280 ent = &queue->qhead->head[queue->idx * OTX_CPT_INST_SIZE];
281 memcpy(ent, (void *) cptinst, OTX_CPT_INST_SIZE);
282
283 if (++queue->idx >= queue->qhead->size / 64) {
284 curr = queue->qhead;
285
286 if (list_is_last(&curr->nextchunk, &queue->chead))
287 queue->qhead = queue->base;
288 else
289 queue->qhead = list_next_entry(queue->qhead, nextchunk);
290 queue->idx = 0;
291 }
292 /* make sure all memory stores are done before ringing doorbell */
293 smp_wmb();
294 otx_cptvf_write_vq_doorbell(cptvf, 1);
295 }
296
process_request(struct pci_dev * pdev,struct otx_cpt_req_info * req,struct otx_cpt_pending_queue * pqueue,struct otx_cptvf * cptvf)297 static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
298 struct otx_cpt_pending_queue *pqueue,
299 struct otx_cptvf *cptvf)
300 {
301 struct otx_cptvf_request *cpt_req = &req->req;
302 struct otx_cpt_pending_entry *pentry = NULL;
303 union otx_cpt_ctrl_info *ctrl = &req->ctrl;
304 struct otx_cpt_info_buffer *info = NULL;
305 union otx_cpt_res_s *result = NULL;
306 struct otx_cpt_iq_cmd iq_cmd;
307 union otx_cpt_inst_s cptinst;
308 int retry, ret = 0;
309 u8 resume_sender;
310 gfp_t gfp;
311
312 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
313 GFP_ATOMIC;
314 ret = setup_sgio_list(pdev, &info, req, gfp);
315 if (unlikely(ret)) {
316 dev_err(&pdev->dev, "Setting up SG list failed\n");
317 goto request_cleanup;
318 }
319 cpt_req->dlen = info->dlen;
320
321 result = (union otx_cpt_res_s *) info->completion_addr;
322 result->s.compcode = COMPLETION_CODE_INIT;
323
324 spin_lock_bh(&pqueue->lock);
325 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
326 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
327 while (unlikely(!pentry) && retry--) {
328 spin_unlock_bh(&pqueue->lock);
329 udelay(CPT_PENTRY_STEP);
330 spin_lock_bh(&pqueue->lock);
331 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
332 }
333
334 if (unlikely(!pentry)) {
335 ret = -ENOSPC;
336 spin_unlock_bh(&pqueue->lock);
337 goto request_cleanup;
338 }
339
340 /*
341 * Check if we are close to filling in entire pending queue,
342 * if so then tell the sender to stop/sleep by returning -EBUSY
343 * We do it only for context which can sleep (GFP_KERNEL)
344 */
345 if (gfp == GFP_KERNEL &&
346 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
347 pentry->resume_sender = true;
348 } else
349 pentry->resume_sender = false;
350 resume_sender = pentry->resume_sender;
351 pqueue->pending_count++;
352
353 pentry->completion_addr = info->completion_addr;
354 pentry->info = info;
355 pentry->callback = req->callback;
356 pentry->areq = req->areq;
357 pentry->busy = true;
358 info->pentry = pentry;
359 info->time_in = jiffies;
360 info->req = req;
361
362 /* Fill in the command */
363 iq_cmd.cmd.u64 = 0;
364 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
365 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
366 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
367 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
368
369 iq_cmd.dptr = info->dptr_baddr;
370 iq_cmd.rptr = info->rptr_baddr;
371 iq_cmd.cptr.u64 = 0;
372 iq_cmd.cptr.s.grp = ctrl->s.grp;
373
374 /* Fill in the CPT_INST_S type command for HW interpretation */
375 cpt_fill_inst(&cptinst, info, &iq_cmd);
376
377 /* Print debug info if enabled */
378 otx_cpt_dump_sg_list(pdev, req);
379 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX_CPT_INST_SIZE);
380 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX_CPT_INST_SIZE, false);
381 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
382 print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
383 cpt_req->dlen, false);
384
385 /* Send CPT command */
386 cpt_send_cmd(&cptinst, cptvf);
387
388 /*
389 * We allocate and prepare pending queue entry in critical section
390 * together with submitting CPT instruction to CPT instruction queue
391 * to make sure that order of CPT requests is the same in both
392 * pending and instruction queues
393 */
394 spin_unlock_bh(&pqueue->lock);
395
396 ret = resume_sender ? -EBUSY : -EINPROGRESS;
397 return ret;
398
399 request_cleanup:
400 do_request_cleanup(pdev, info);
401 return ret;
402 }
403
otx_cpt_do_request(struct pci_dev * pdev,struct otx_cpt_req_info * req,int cpu_num)404 int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req,
405 int cpu_num)
406 {
407 struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
408
409 if (!otx_cpt_device_ready(cptvf)) {
410 dev_err(&pdev->dev, "CPT Device is not ready\n");
411 return -ENODEV;
412 }
413
414 if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) {
415 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n",
416 cptvf->vfid);
417 return -EINVAL;
418 } else if ((cptvf->vftype == OTX_CPT_AE_TYPES) &&
419 (req->ctrl.s.se_req)) {
420 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n",
421 cptvf->vfid);
422 return -EINVAL;
423 }
424
425 return process_request(pdev, req, &cptvf->pqinfo.queue[0], cptvf);
426 }
427
cpt_process_ccode(struct pci_dev * pdev,union otx_cpt_res_s * cpt_status,struct otx_cpt_info_buffer * cpt_info,struct otx_cpt_req_info * req,u32 * res_code)428 static int cpt_process_ccode(struct pci_dev *pdev,
429 union otx_cpt_res_s *cpt_status,
430 struct otx_cpt_info_buffer *cpt_info,
431 struct otx_cpt_req_info *req, u32 *res_code)
432 {
433 u8 ccode = cpt_status->s.compcode;
434 union otx_cpt_error_code ecode;
435
436 ecode.u = be64_to_cpup((__be64 *)cpt_info->out_buffer);
437 switch (ccode) {
438 case CPT_COMP_E_FAULT:
439 dev_err(&pdev->dev,
440 "Request failed with DMA fault\n");
441 otx_cpt_dump_sg_list(pdev, req);
442 break;
443
444 case CPT_COMP_E_SWERR:
445 dev_err(&pdev->dev,
446 "Request failed with software error code %d\n",
447 ecode.s.ccode);
448 otx_cpt_dump_sg_list(pdev, req);
449 break;
450
451 case CPT_COMP_E_HWERR:
452 dev_err(&pdev->dev,
453 "Request failed with hardware error\n");
454 otx_cpt_dump_sg_list(pdev, req);
455 break;
456
457 case COMPLETION_CODE_INIT:
458 /* check for timeout */
459 if (time_after_eq(jiffies, cpt_info->time_in +
460 OTX_CPT_COMMAND_TIMEOUT * HZ))
461 dev_warn(&pdev->dev, "Request timed out 0x%p\n", req);
462 else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) {
463 cpt_info->time_in = jiffies;
464 cpt_info->extra_time++;
465 }
466 return 1;
467
468 case CPT_COMP_E_GOOD:
469 /* Check microcode completion code */
470 if (ecode.s.ccode) {
471 /*
472 * If requested hmac is truncated and ucode returns
473 * s/g write length error then we report success
474 * because ucode writes as many bytes of calculated
475 * hmac as available in gather buffer and reports
476 * s/g write length error if number of bytes in gather
477 * buffer is less than full hmac size.
478 */
479 if (req->is_trunc_hmac &&
480 ecode.s.ccode == ERR_SCATTER_GATHER_WRITE_LENGTH) {
481 *res_code = 0;
482 break;
483 }
484
485 dev_err(&pdev->dev,
486 "Request failed with software error code 0x%x\n",
487 ecode.s.ccode);
488 otx_cpt_dump_sg_list(pdev, req);
489 break;
490 }
491
492 /* Request has been processed with success */
493 *res_code = 0;
494 break;
495
496 default:
497 dev_err(&pdev->dev, "Request returned invalid status\n");
498 break;
499 }
500
501 return 0;
502 }
503
process_pending_queue(struct pci_dev * pdev,struct otx_cpt_pending_queue * pqueue)504 static inline void process_pending_queue(struct pci_dev *pdev,
505 struct otx_cpt_pending_queue *pqueue)
506 {
507 void (*callback)(int status, void *arg1, void *arg2);
508 struct otx_cpt_pending_entry *resume_pentry = NULL;
509 struct otx_cpt_pending_entry *pentry = NULL;
510 struct otx_cpt_info_buffer *cpt_info = NULL;
511 union otx_cpt_res_s *cpt_status = NULL;
512 struct otx_cpt_req_info *req = NULL;
513 struct crypto_async_request *areq;
514 u32 res_code, resume_index;
515
516 while (1) {
517 spin_lock_bh(&pqueue->lock);
518 pentry = &pqueue->head[pqueue->front];
519
520 if (WARN_ON(!pentry)) {
521 spin_unlock_bh(&pqueue->lock);
522 break;
523 }
524
525 res_code = -EINVAL;
526 if (unlikely(!pentry->busy)) {
527 spin_unlock_bh(&pqueue->lock);
528 break;
529 }
530
531 if (unlikely(!pentry->callback)) {
532 dev_err(&pdev->dev, "Callback NULL\n");
533 goto process_pentry;
534 }
535
536 cpt_info = pentry->info;
537 if (unlikely(!cpt_info)) {
538 dev_err(&pdev->dev, "Pending entry post arg NULL\n");
539 goto process_pentry;
540 }
541
542 req = cpt_info->req;
543 if (unlikely(!req)) {
544 dev_err(&pdev->dev, "Request NULL\n");
545 goto process_pentry;
546 }
547
548 cpt_status = (union otx_cpt_res_s *) pentry->completion_addr;
549 if (unlikely(!cpt_status)) {
550 dev_err(&pdev->dev, "Completion address NULL\n");
551 goto process_pentry;
552 }
553
554 if (cpt_process_ccode(pdev, cpt_status, cpt_info, req,
555 &res_code)) {
556 spin_unlock_bh(&pqueue->lock);
557 return;
558 }
559 cpt_info->pdev = pdev;
560
561 process_pentry:
562 /*
563 * Check if we should inform sending side to resume
564 * We do it CPT_IQ_RESUME_MARGIN elements in advance before
565 * pending queue becomes empty
566 */
567 resume_index = modulo_inc(pqueue->front, pqueue->qlen,
568 CPT_IQ_RESUME_MARGIN);
569 resume_pentry = &pqueue->head[resume_index];
570 if (resume_pentry &&
571 resume_pentry->resume_sender) {
572 resume_pentry->resume_sender = false;
573 callback = resume_pentry->callback;
574 areq = resume_pentry->areq;
575
576 if (callback) {
577 spin_unlock_bh(&pqueue->lock);
578
579 /*
580 * EINPROGRESS is an indication for sending
581 * side that it can resume sending requests
582 */
583 callback(-EINPROGRESS, areq, cpt_info);
584 spin_lock_bh(&pqueue->lock);
585 }
586 }
587
588 callback = pentry->callback;
589 areq = pentry->areq;
590 free_pentry(pentry);
591
592 pqueue->pending_count--;
593 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
594 spin_unlock_bh(&pqueue->lock);
595
596 /*
597 * Call callback after current pending entry has been
598 * processed, we don't do it if the callback pointer is
599 * invalid.
600 */
601 if (callback)
602 callback(res_code, areq, cpt_info);
603 }
604 }
605
otx_cpt_post_process(struct otx_cptvf_wqe * wqe)606 void otx_cpt_post_process(struct otx_cptvf_wqe *wqe)
607 {
608 process_pending_queue(wqe->cptvf->pdev, &wqe->cptvf->pqinfo.queue[0]);
609 }
610