1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2016 Cavium, Inc.
4 */
5
6 #include "cptvf.h"
7 #include "cptvf_algs.h"
8 #include "request_manager.h"
9
10 /**
11 * get_free_pending_entry - get free entry from pending queue
12 * @q: pending queue
13 * @qlen: queue length
14 */
get_free_pending_entry(struct pending_queue * q,int qlen)15 static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
16 int qlen)
17 {
18 struct pending_entry *ent = NULL;
19
20 ent = &q->head[q->rear];
21 if (unlikely(ent->busy)) {
22 ent = NULL;
23 goto no_free_entry;
24 }
25
26 q->rear++;
27 if (unlikely(q->rear == qlen))
28 q->rear = 0;
29
30 no_free_entry:
31 return ent;
32 }
33
pending_queue_inc_front(struct pending_qinfo * pqinfo,int qno)34 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
35 int qno)
36 {
37 struct pending_queue *queue = &pqinfo->queue[qno];
38
39 queue->front++;
40 if (unlikely(queue->front == pqinfo->qlen))
41 queue->front = 0;
42 }
43
setup_sgio_components(struct cpt_vf * cptvf,struct buf_ptr * list,int buf_count,u8 * buffer)44 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
45 int buf_count, u8 *buffer)
46 {
47 int ret = 0, i, j;
48 int components;
49 struct sglist_component *sg_ptr = NULL;
50 struct pci_dev *pdev = cptvf->pdev;
51
52 if (unlikely(!list)) {
53 dev_err(&pdev->dev, "Input List pointer is NULL\n");
54 return -EFAULT;
55 }
56
57 for (i = 0; i < buf_count; i++) {
58 if (likely(list[i].vptr)) {
59 list[i].dma_addr = dma_map_single(&pdev->dev,
60 list[i].vptr,
61 list[i].size,
62 DMA_BIDIRECTIONAL);
63 if (unlikely(dma_mapping_error(&pdev->dev,
64 list[i].dma_addr))) {
65 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
66 i);
67 ret = -EIO;
68 goto sg_cleanup;
69 }
70 }
71 }
72
73 components = buf_count / 4;
74 sg_ptr = (struct sglist_component *)buffer;
75 for (i = 0; i < components; i++) {
76 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
77 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
78 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
79 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
80 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
81 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
82 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
83 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
84 sg_ptr++;
85 }
86
87 components = buf_count % 4;
88
89 switch (components) {
90 case 3:
91 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
92 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
93 fallthrough;
94 case 2:
95 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
96 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
97 fallthrough;
98 case 1:
99 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
100 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
101 break;
102 default:
103 break;
104 }
105
106 return ret;
107
108 sg_cleanup:
109 for (j = 0; j < i; j++) {
110 if (list[j].dma_addr) {
111 dma_unmap_single(&pdev->dev, list[i].dma_addr,
112 list[i].size, DMA_BIDIRECTIONAL);
113 }
114
115 list[j].dma_addr = 0;
116 }
117
118 return ret;
119 }
120
setup_sgio_list(struct cpt_vf * cptvf,struct cpt_info_buffer * info,struct cpt_request_info * req)121 static inline int setup_sgio_list(struct cpt_vf *cptvf,
122 struct cpt_info_buffer *info,
123 struct cpt_request_info *req)
124 {
125 u16 g_sz_bytes = 0, s_sz_bytes = 0;
126 int ret = 0;
127 struct pci_dev *pdev = cptvf->pdev;
128
129 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
130 dev_err(&pdev->dev, "Request SG components are higher than supported\n");
131 ret = -EINVAL;
132 goto scatter_gather_clean;
133 }
134
135 /* Setup gather (input) components */
136 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
137 info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
138 if (!info->gather_components) {
139 ret = -ENOMEM;
140 goto scatter_gather_clean;
141 }
142
143 ret = setup_sgio_components(cptvf, req->in,
144 req->incnt,
145 info->gather_components);
146 if (ret) {
147 dev_err(&pdev->dev, "Failed to setup gather list\n");
148 ret = -EFAULT;
149 goto scatter_gather_clean;
150 }
151
152 /* Setup scatter (output) components */
153 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
154 info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
155 if (!info->scatter_components) {
156 ret = -ENOMEM;
157 goto scatter_gather_clean;
158 }
159
160 ret = setup_sgio_components(cptvf, req->out,
161 req->outcnt,
162 info->scatter_components);
163 if (ret) {
164 dev_err(&pdev->dev, "Failed to setup gather list\n");
165 ret = -EFAULT;
166 goto scatter_gather_clean;
167 }
168
169 /* Create and initialize DPTR */
170 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
171 info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
172 if (!info->in_buffer) {
173 ret = -ENOMEM;
174 goto scatter_gather_clean;
175 }
176
177 ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
178 ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
179 ((__be16 *)info->in_buffer)[2] = 0;
180 ((__be16 *)info->in_buffer)[3] = 0;
181
182 memcpy(&info->in_buffer[8], info->gather_components,
183 g_sz_bytes);
184 memcpy(&info->in_buffer[8 + g_sz_bytes],
185 info->scatter_components, s_sz_bytes);
186
187 info->dptr_baddr = dma_map_single(&pdev->dev,
188 (void *)info->in_buffer,
189 info->dlen,
190 DMA_BIDIRECTIONAL);
191 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
192 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
193 ret = -EIO;
194 goto scatter_gather_clean;
195 }
196
197 /* Create and initialize RPTR */
198 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
199 if (!info->out_buffer) {
200 ret = -ENOMEM;
201 goto scatter_gather_clean;
202 }
203
204 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
205 info->alternate_caddr = (u64 *)info->out_buffer;
206 info->rptr_baddr = dma_map_single(&pdev->dev,
207 (void *)info->out_buffer,
208 COMPLETION_CODE_SIZE,
209 DMA_BIDIRECTIONAL);
210 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
211 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
212 COMPLETION_CODE_SIZE);
213 ret = -EIO;
214 goto scatter_gather_clean;
215 }
216
217 return 0;
218
219 scatter_gather_clean:
220 return ret;
221 }
222
send_cpt_command(struct cpt_vf * cptvf,union cpt_inst_s * cmd,u32 qno)223 static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
224 u32 qno)
225 {
226 struct pci_dev *pdev = cptvf->pdev;
227 struct command_qinfo *qinfo = NULL;
228 struct command_queue *queue;
229 struct command_chunk *chunk;
230 u8 *ent;
231 int ret = 0;
232
233 if (unlikely(qno >= cptvf->nr_queues)) {
234 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
235 qno, cptvf->nr_queues);
236 return -EINVAL;
237 }
238
239 qinfo = &cptvf->cqinfo;
240 queue = &qinfo->queue[qno];
241 /* lock commad queue */
242 spin_lock(&queue->lock);
243 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
244 memcpy(ent, (void *)cmd, qinfo->cmd_size);
245
246 if (++queue->idx >= queue->qhead->size / 64) {
247 hlist_for_each_entry(chunk, &queue->chead, nextchunk) {
248 if (chunk == queue->qhead) {
249 continue;
250 } else {
251 queue->qhead = chunk;
252 break;
253 }
254 }
255 queue->idx = 0;
256 }
257 /* make sure all memory stores are done before ringing doorbell */
258 smp_wmb();
259 cptvf_write_vq_doorbell(cptvf, 1);
260 /* unlock command queue */
261 spin_unlock(&queue->lock);
262
263 return ret;
264 }
265
do_request_cleanup(struct cpt_vf * cptvf,struct cpt_info_buffer * info)266 static void do_request_cleanup(struct cpt_vf *cptvf,
267 struct cpt_info_buffer *info)
268 {
269 int i;
270 struct pci_dev *pdev = cptvf->pdev;
271 struct cpt_request_info *req;
272
273 if (info->dptr_baddr)
274 dma_unmap_single(&pdev->dev, info->dptr_baddr,
275 info->dlen, DMA_BIDIRECTIONAL);
276
277 if (info->rptr_baddr)
278 dma_unmap_single(&pdev->dev, info->rptr_baddr,
279 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
280
281 if (info->comp_baddr)
282 dma_unmap_single(&pdev->dev, info->comp_baddr,
283 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
284
285 if (info->req) {
286 req = info->req;
287 for (i = 0; i < req->outcnt; i++) {
288 if (req->out[i].dma_addr)
289 dma_unmap_single(&pdev->dev,
290 req->out[i].dma_addr,
291 req->out[i].size,
292 DMA_BIDIRECTIONAL);
293 }
294
295 for (i = 0; i < req->incnt; i++) {
296 if (req->in[i].dma_addr)
297 dma_unmap_single(&pdev->dev,
298 req->in[i].dma_addr,
299 req->in[i].size,
300 DMA_BIDIRECTIONAL);
301 }
302 }
303
304 kfree_sensitive(info->scatter_components);
305 kfree_sensitive(info->gather_components);
306 kfree_sensitive(info->out_buffer);
307 kfree_sensitive(info->in_buffer);
308 kfree_sensitive((void *)info->completion_addr);
309 kfree_sensitive(info);
310 }
311
do_post_process(struct cpt_vf * cptvf,struct cpt_info_buffer * info)312 static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
313 {
314 struct pci_dev *pdev = cptvf->pdev;
315
316 if (!info) {
317 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
318 return;
319 }
320
321 do_request_cleanup(cptvf, info);
322 }
323
process_pending_queue(struct cpt_vf * cptvf,struct pending_qinfo * pqinfo,int qno)324 static inline void process_pending_queue(struct cpt_vf *cptvf,
325 struct pending_qinfo *pqinfo,
326 int qno)
327 {
328 struct pci_dev *pdev = cptvf->pdev;
329 struct pending_queue *pqueue = &pqinfo->queue[qno];
330 struct pending_entry *pentry = NULL;
331 struct cpt_info_buffer *info = NULL;
332 union cpt_res_s *status = NULL;
333 unsigned char ccode;
334
335 while (1) {
336 spin_lock_bh(&pqueue->lock);
337 pentry = &pqueue->head[pqueue->front];
338 if (unlikely(!pentry->busy)) {
339 spin_unlock_bh(&pqueue->lock);
340 break;
341 }
342
343 info = (struct cpt_info_buffer *)pentry->post_arg;
344 if (unlikely(!info)) {
345 dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
346 pending_queue_inc_front(pqinfo, qno);
347 spin_unlock_bh(&pqueue->lock);
348 continue;
349 }
350
351 status = (union cpt_res_s *)pentry->completion_addr;
352 ccode = status->s.compcode;
353 if ((status->s.compcode == CPT_COMP_E_FAULT) ||
354 (status->s.compcode == CPT_COMP_E_SWERR)) {
355 dev_err(&pdev->dev, "Request failed with %s\n",
356 (status->s.compcode == CPT_COMP_E_FAULT) ?
357 "DMA Fault" : "Software error");
358 pentry->completion_addr = NULL;
359 pentry->busy = false;
360 atomic64_dec((&pqueue->pending_count));
361 pentry->post_arg = NULL;
362 pending_queue_inc_front(pqinfo, qno);
363 do_request_cleanup(cptvf, info);
364 spin_unlock_bh(&pqueue->lock);
365 break;
366 } else if (status->s.compcode == COMPLETION_CODE_INIT) {
367 /* check for timeout */
368 if (time_after_eq(jiffies,
369 (info->time_in +
370 (CPT_COMMAND_TIMEOUT * HZ)))) {
371 dev_err(&pdev->dev, "Request timed out");
372 pentry->completion_addr = NULL;
373 pentry->busy = false;
374 atomic64_dec((&pqueue->pending_count));
375 pentry->post_arg = NULL;
376 pending_queue_inc_front(pqinfo, qno);
377 do_request_cleanup(cptvf, info);
378 spin_unlock_bh(&pqueue->lock);
379 break;
380 } else if ((*info->alternate_caddr ==
381 (~COMPLETION_CODE_INIT)) &&
382 (info->extra_time < TIME_IN_RESET_COUNT)) {
383 info->time_in = jiffies;
384 info->extra_time++;
385 spin_unlock_bh(&pqueue->lock);
386 break;
387 }
388 }
389
390 pentry->completion_addr = NULL;
391 pentry->busy = false;
392 pentry->post_arg = NULL;
393 atomic64_dec((&pqueue->pending_count));
394 pending_queue_inc_front(pqinfo, qno);
395 spin_unlock_bh(&pqueue->lock);
396
397 do_post_process(info->cptvf, info);
398 /*
399 * Calling callback after we find
400 * that the request has been serviced
401 */
402 pentry->callback(ccode, pentry->callback_arg);
403 }
404 }
405
process_request(struct cpt_vf * cptvf,struct cpt_request_info * req)406 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
407 {
408 int ret = 0, clear = 0, queue = 0;
409 struct cpt_info_buffer *info = NULL;
410 struct cptvf_request *cpt_req = NULL;
411 union ctrl_info *ctrl = NULL;
412 union cpt_res_s *result = NULL;
413 struct pending_entry *pentry = NULL;
414 struct pending_queue *pqueue = NULL;
415 struct pci_dev *pdev = cptvf->pdev;
416 u8 group = 0;
417 struct cpt_vq_command vq_cmd;
418 union cpt_inst_s cptinst;
419
420 info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
421 if (unlikely(!info)) {
422 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
423 return -ENOMEM;
424 }
425
426 cpt_req = (struct cptvf_request *)&req->req;
427 ctrl = (union ctrl_info *)&req->ctrl;
428
429 info->cptvf = cptvf;
430 group = ctrl->s.grp;
431 ret = setup_sgio_list(cptvf, info, req);
432 if (ret) {
433 dev_err(&pdev->dev, "Setting up SG list failed");
434 goto request_cleanup;
435 }
436
437 cpt_req->dlen = info->dlen;
438 /*
439 * Get buffer for union cpt_res_s response
440 * structure and its physical address
441 */
442 info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
443 if (unlikely(!info->completion_addr)) {
444 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
445 ret = -ENOMEM;
446 goto request_cleanup;
447 }
448
449 result = (union cpt_res_s *)info->completion_addr;
450 result->s.compcode = COMPLETION_CODE_INIT;
451 info->comp_baddr = dma_map_single(&pdev->dev,
452 (void *)info->completion_addr,
453 sizeof(union cpt_res_s),
454 DMA_BIDIRECTIONAL);
455 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
456 dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
457 sizeof(union cpt_res_s));
458 ret = -EFAULT;
459 goto request_cleanup;
460 }
461
462 /* Fill the VQ command */
463 vq_cmd.cmd.u64 = 0;
464 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
465 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
466 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
467 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
468
469 vq_cmd.dptr = info->dptr_baddr;
470 vq_cmd.rptr = info->rptr_baddr;
471 vq_cmd.cptr.u64 = 0;
472 vq_cmd.cptr.s.grp = group;
473 /* Get Pending Entry to submit command */
474 /* Always queue 0, because 1 queue per VF */
475 queue = 0;
476 pqueue = &cptvf->pqinfo.queue[queue];
477
478 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
479 dev_err(&pdev->dev, "pending threshold reached\n");
480 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
481 }
482
483 get_pending_entry:
484 spin_lock_bh(&pqueue->lock);
485 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
486 if (unlikely(!pentry)) {
487 spin_unlock_bh(&pqueue->lock);
488 if (clear == 0) {
489 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
490 clear = 1;
491 goto get_pending_entry;
492 }
493 dev_err(&pdev->dev, "Get free entry failed\n");
494 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
495 queue, pqueue->rear, pqueue->front);
496 ret = -EFAULT;
497 goto request_cleanup;
498 }
499
500 pentry->completion_addr = info->completion_addr;
501 pentry->post_arg = (void *)info;
502 pentry->callback = req->callback;
503 pentry->callback_arg = req->callback_arg;
504 info->pentry = pentry;
505 pentry->busy = true;
506 atomic64_inc(&pqueue->pending_count);
507
508 /* Send CPT command */
509 info->pentry = pentry;
510 info->time_in = jiffies;
511 info->req = req;
512
513 /* Create the CPT_INST_S type command for HW intrepretation */
514 cptinst.s.doneint = true;
515 cptinst.s.res_addr = (u64)info->comp_baddr;
516 cptinst.s.tag = 0;
517 cptinst.s.grp = 0;
518 cptinst.s.wq_ptr = 0;
519 cptinst.s.ei0 = vq_cmd.cmd.u64;
520 cptinst.s.ei1 = vq_cmd.dptr;
521 cptinst.s.ei2 = vq_cmd.rptr;
522 cptinst.s.ei3 = vq_cmd.cptr.u64;
523
524 ret = send_cpt_command(cptvf, &cptinst, queue);
525 spin_unlock_bh(&pqueue->lock);
526 if (unlikely(ret)) {
527 dev_err(&pdev->dev, "Send command failed for AE\n");
528 ret = -EFAULT;
529 goto request_cleanup;
530 }
531
532 return 0;
533
534 request_cleanup:
535 dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
536 do_request_cleanup(cptvf, info);
537
538 return ret;
539 }
540
vq_post_process(struct cpt_vf * cptvf,u32 qno)541 void vq_post_process(struct cpt_vf *cptvf, u32 qno)
542 {
543 struct pci_dev *pdev = cptvf->pdev;
544
545 if (unlikely(qno > cptvf->nr_queues)) {
546 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
547 qno);
548 return;
549 }
550
551 process_pending_queue(cptvf, &cptvf->pqinfo, qno);
552 }
553
cptvf_do_request(void * vfdev,struct cpt_request_info * req)554 int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
555 {
556 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
557 struct pci_dev *pdev = cptvf->pdev;
558
559 if (!cpt_device_ready(cptvf)) {
560 dev_err(&pdev->dev, "CPT Device is not ready");
561 return -ENODEV;
562 }
563
564 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
565 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
566 cptvf->vfid);
567 return -EINVAL;
568 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
569 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
570 cptvf->vfid);
571 return -EINVAL;
572 }
573
574 return process_request(cptvf, req);
575 }
576