1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright 2019 NXP
3
4 #include <linux/init.h>
5 #include <linux/module.h>
6 #include <linux/dmapool.h>
7 #include <linux/of_irq.h>
8 #include <linux/iommu.h>
9 #include <linux/sys_soc.h>
10 #include <linux/fsl/mc.h>
11 #include <soc/fsl/dpaa2-io.h>
12
13 #include "../virt-dma.h"
14 #include "dpdmai.h"
15 #include "dpaa2-qdma.h"
16
17 static bool smmu_disable = true;
18
to_dpaa2_qdma_chan(struct dma_chan * chan)19 static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
20 {
21 return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
22 }
23
to_fsl_qdma_comp(struct virt_dma_desc * vd)24 static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
25 {
26 return container_of(vd, struct dpaa2_qdma_comp, vdesc);
27 }
28
dpaa2_qdma_alloc_chan_resources(struct dma_chan * chan)29 static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
30 {
31 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
32 struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
33 struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
34
35 dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev,
36 sizeof(struct dpaa2_fd),
37 sizeof(struct dpaa2_fd), 0);
38 if (!dpaa2_chan->fd_pool)
39 goto err;
40
41 dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev,
42 sizeof(struct dpaa2_fl_entry),
43 sizeof(struct dpaa2_fl_entry), 0);
44 if (!dpaa2_chan->fl_pool)
45 goto err_fd;
46
47 dpaa2_chan->sdd_pool =
48 dma_pool_create("sdd_pool", dev,
49 sizeof(struct dpaa2_qdma_sd_d),
50 sizeof(struct dpaa2_qdma_sd_d), 0);
51 if (!dpaa2_chan->sdd_pool)
52 goto err_fl;
53
54 return dpaa2_qdma->desc_allocated++;
55 err_fl:
56 dma_pool_destroy(dpaa2_chan->fl_pool);
57 err_fd:
58 dma_pool_destroy(dpaa2_chan->fd_pool);
59 err:
60 return -ENOMEM;
61 }
62
dpaa2_qdma_free_chan_resources(struct dma_chan * chan)63 static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
64 {
65 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
66 struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
67 unsigned long flags;
68
69 LIST_HEAD(head);
70
71 spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
72 vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
73 spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
74
75 vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
76
77 dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used);
78 dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free);
79
80 dma_pool_destroy(dpaa2_chan->fd_pool);
81 dma_pool_destroy(dpaa2_chan->fl_pool);
82 dma_pool_destroy(dpaa2_chan->sdd_pool);
83 dpaa2_qdma->desc_allocated--;
84 }
85
86 /*
87 * Request a command descriptor for enqueue.
88 */
89 static struct dpaa2_qdma_comp *
dpaa2_qdma_request_desc(struct dpaa2_qdma_chan * dpaa2_chan)90 dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
91 {
92 struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv;
93 struct device *dev = &qdma_priv->dpdmai_dev->dev;
94 struct dpaa2_qdma_comp *comp_temp = NULL;
95 unsigned long flags;
96
97 spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
98 if (list_empty(&dpaa2_chan->comp_free)) {
99 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
100 comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT);
101 if (!comp_temp)
102 goto err;
103 comp_temp->fd_virt_addr =
104 dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT,
105 &comp_temp->fd_bus_addr);
106 if (!comp_temp->fd_virt_addr)
107 goto err_comp;
108
109 comp_temp->fl_virt_addr =
110 dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT,
111 &comp_temp->fl_bus_addr);
112 if (!comp_temp->fl_virt_addr)
113 goto err_fd_virt;
114
115 comp_temp->desc_virt_addr =
116 dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT,
117 &comp_temp->desc_bus_addr);
118 if (!comp_temp->desc_virt_addr)
119 goto err_fl_virt;
120
121 comp_temp->qchan = dpaa2_chan;
122 return comp_temp;
123 }
124
125 comp_temp = list_first_entry(&dpaa2_chan->comp_free,
126 struct dpaa2_qdma_comp, list);
127 list_del(&comp_temp->list);
128 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
129
130 comp_temp->qchan = dpaa2_chan;
131
132 return comp_temp;
133
134 err_fl_virt:
135 dma_pool_free(dpaa2_chan->fl_pool,
136 comp_temp->fl_virt_addr,
137 comp_temp->fl_bus_addr);
138 err_fd_virt:
139 dma_pool_free(dpaa2_chan->fd_pool,
140 comp_temp->fd_virt_addr,
141 comp_temp->fd_bus_addr);
142 err_comp:
143 kfree(comp_temp);
144 err:
145 dev_err(dev, "Failed to request descriptor\n");
146 return NULL;
147 }
148
149 static void
dpaa2_qdma_populate_fd(u32 format,struct dpaa2_qdma_comp * dpaa2_comp)150 dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp)
151 {
152 struct dpaa2_fd *fd;
153
154 fd = dpaa2_comp->fd_virt_addr;
155 memset(fd, 0, sizeof(struct dpaa2_fd));
156
157 /* fd populated */
158 dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);
159
160 /*
161 * Bypass memory translation, Frame list format, short length disable
162 * we need to disable BMT if fsl-mc use iova addr
163 */
164 if (smmu_disable)
165 dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);
166 dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);
167
168 dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);
169 }
170
171 /* first frame list for descriptor buffer */
172 static void
dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry * f_list,struct dpaa2_qdma_comp * dpaa2_comp,bool wrt_changed)173 dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list,
174 struct dpaa2_qdma_comp *dpaa2_comp,
175 bool wrt_changed)
176 {
177 struct dpaa2_qdma_sd_d *sdd;
178
179 sdd = dpaa2_comp->desc_virt_addr;
180 memset(sdd, 0, 2 * (sizeof(*sdd)));
181
182 /* source descriptor CMD */
183 sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT);
184 sdd++;
185
186 /* dest descriptor CMD */
187 if (wrt_changed)
188 sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT);
189 else
190 sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT);
191
192 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
193
194 /* first frame list to source descriptor */
195 dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);
196 dpaa2_fl_set_len(f_list, 0x20);
197 dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);
198
199 /* bypass memory translation */
200 if (smmu_disable)
201 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
202 }
203
204 /* source and destination frame list */
205 static void
dpaa2_qdma_populate_frames(struct dpaa2_fl_entry * f_list,dma_addr_t dst,dma_addr_t src,size_t len,uint8_t fmt)206 dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,
207 dma_addr_t dst, dma_addr_t src,
208 size_t len, uint8_t fmt)
209 {
210 /* source frame list to source buffer */
211 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
212
213 dpaa2_fl_set_addr(f_list, src);
214 dpaa2_fl_set_len(f_list, len);
215
216 /* single buffer frame or scatter gather frame */
217 dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
218
219 /* bypass memory translation */
220 if (smmu_disable)
221 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
222
223 f_list++;
224
225 /* destination frame list to destination buffer */
226 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
227
228 dpaa2_fl_set_addr(f_list, dst);
229 dpaa2_fl_set_len(f_list, len);
230 dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
231 /* single buffer frame or scatter gather frame */
232 dpaa2_fl_set_final(f_list, QDMA_FL_F);
233 /* bypass memory translation */
234 if (smmu_disable)
235 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
236 }
237
238 static struct dma_async_tx_descriptor
dpaa2_qdma_prep_memcpy(struct dma_chan * chan,dma_addr_t dst,dma_addr_t src,size_t len,ulong flags)239 *dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
240 dma_addr_t src, size_t len, ulong flags)
241 {
242 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
243 struct dpaa2_qdma_engine *dpaa2_qdma;
244 struct dpaa2_qdma_comp *dpaa2_comp;
245 struct dpaa2_fl_entry *f_list;
246 bool wrt_changed;
247
248 dpaa2_qdma = dpaa2_chan->qdma;
249 dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
250 if (!dpaa2_comp)
251 return NULL;
252
253 wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup;
254
255 /* populate Frame descriptor */
256 dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp);
257
258 f_list = dpaa2_comp->fl_virt_addr;
259
260 /* first frame list for descriptor buffer (logn format) */
261 dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed);
262
263 f_list++;
264
265 dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
266
267 return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
268 }
269
dpaa2_qdma_issue_pending(struct dma_chan * chan)270 static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
271 {
272 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
273 struct dpaa2_qdma_comp *dpaa2_comp;
274 struct virt_dma_desc *vdesc;
275 struct dpaa2_fd *fd;
276 unsigned long flags;
277 int err;
278
279 spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
280 spin_lock(&dpaa2_chan->vchan.lock);
281 if (vchan_issue_pending(&dpaa2_chan->vchan)) {
282 vdesc = vchan_next_desc(&dpaa2_chan->vchan);
283 if (!vdesc)
284 goto err_enqueue;
285 dpaa2_comp = to_fsl_qdma_comp(vdesc);
286
287 fd = dpaa2_comp->fd_virt_addr;
288
289 list_del(&vdesc->node);
290 list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
291
292 err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd);
293 if (err) {
294 list_del(&dpaa2_comp->list);
295 list_add_tail(&dpaa2_comp->list,
296 &dpaa2_chan->comp_free);
297 }
298 }
299 err_enqueue:
300 spin_unlock(&dpaa2_chan->vchan.lock);
301 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
302 }
303
dpaa2_qdma_setup(struct fsl_mc_device * ls_dev)304 static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
305 {
306 struct dpaa2_qdma_priv_per_prio *ppriv;
307 struct device *dev = &ls_dev->dev;
308 struct dpaa2_qdma_priv *priv;
309 u8 prio_def = DPDMAI_PRIO_NUM;
310 int err = -EINVAL;
311 int i;
312
313 priv = dev_get_drvdata(dev);
314
315 priv->dev = dev;
316 priv->dpqdma_id = ls_dev->obj_desc.id;
317
318 /* Get the handle for the DPDMAI this interface is associate with */
319 err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
320 if (err) {
321 dev_err(dev, "dpdmai_open() failed\n");
322 return err;
323 }
324
325 dev_dbg(dev, "Opened dpdmai object successfully\n");
326
327 err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
328 &priv->dpdmai_attr);
329 if (err) {
330 dev_err(dev, "dpdmai_get_attributes() failed\n");
331 goto exit;
332 }
333
334 if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
335 err = -EINVAL;
336 dev_err(dev, "DPDMAI major version mismatch\n"
337 "Found %u.%u, supported version is %u.%u\n",
338 priv->dpdmai_attr.version.major,
339 priv->dpdmai_attr.version.minor,
340 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
341 goto exit;
342 }
343
344 if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
345 err = -EINVAL;
346 dev_err(dev, "DPDMAI minor version mismatch\n"
347 "Found %u.%u, supported version is %u.%u\n",
348 priv->dpdmai_attr.version.major,
349 priv->dpdmai_attr.version.minor,
350 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
351 goto exit;
352 }
353
354 priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
355 ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
356 if (!ppriv) {
357 err = -ENOMEM;
358 goto exit;
359 }
360 priv->ppriv = ppriv;
361
362 for (i = 0; i < priv->num_pairs; i++) {
363 err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
364 i, &priv->rx_queue_attr[i]);
365 if (err) {
366 dev_err(dev, "dpdmai_get_rx_queue() failed\n");
367 goto exit;
368 }
369 ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
370
371 err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
372 i, &priv->tx_fqid[i]);
373 if (err) {
374 dev_err(dev, "dpdmai_get_tx_queue() failed\n");
375 goto exit;
376 }
377 ppriv->req_fqid = priv->tx_fqid[i];
378 ppriv->prio = i;
379 ppriv->priv = priv;
380 ppriv++;
381 }
382
383 return 0;
384 exit:
385 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
386 return err;
387 }
388
dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx * ctx)389 static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
390 {
391 struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
392 struct dpaa2_qdma_priv_per_prio, nctx);
393 struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
394 struct dpaa2_qdma_priv *priv = ppriv->priv;
395 u32 n_chans = priv->dpaa2_qdma->n_chans;
396 struct dpaa2_qdma_chan *qchan;
397 const struct dpaa2_fd *fd_eq;
398 const struct dpaa2_fd *fd;
399 struct dpaa2_dq *dq;
400 int is_last = 0;
401 int found;
402 u8 status;
403 int err;
404 int i;
405
406 do {
407 err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
408 ppriv->store);
409 } while (err);
410
411 while (!is_last) {
412 do {
413 dq = dpaa2_io_store_next(ppriv->store, &is_last);
414 } while (!is_last && !dq);
415 if (!dq) {
416 dev_err(priv->dev, "FQID returned no valid frames!\n");
417 continue;
418 }
419
420 /* obtain FD and process the error */
421 fd = dpaa2_dq_fd(dq);
422
423 status = dpaa2_fd_get_ctrl(fd) & 0xff;
424 if (status)
425 dev_err(priv->dev, "FD error occurred\n");
426 found = 0;
427 for (i = 0; i < n_chans; i++) {
428 qchan = &priv->dpaa2_qdma->chans[i];
429 spin_lock(&qchan->queue_lock);
430 if (list_empty(&qchan->comp_used)) {
431 spin_unlock(&qchan->queue_lock);
432 continue;
433 }
434 list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
435 &qchan->comp_used, list) {
436 fd_eq = dpaa2_comp->fd_virt_addr;
437
438 if (le64_to_cpu(fd_eq->simple.addr) ==
439 le64_to_cpu(fd->simple.addr)) {
440 spin_lock(&qchan->vchan.lock);
441 vchan_cookie_complete(&
442 dpaa2_comp->vdesc);
443 spin_unlock(&qchan->vchan.lock);
444 found = 1;
445 break;
446 }
447 }
448 spin_unlock(&qchan->queue_lock);
449 if (found)
450 break;
451 }
452 }
453
454 dpaa2_io_service_rearm(NULL, ctx);
455 }
456
dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv * priv)457 static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
458 {
459 struct dpaa2_qdma_priv_per_prio *ppriv;
460 struct device *dev = priv->dev;
461 int err = -EINVAL;
462 int i, num;
463
464 num = priv->num_pairs;
465 ppriv = priv->ppriv;
466 for (i = 0; i < num; i++) {
467 ppriv->nctx.is_cdan = 0;
468 ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU;
469 ppriv->nctx.id = ppriv->rsp_fqid;
470 ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
471 err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev);
472 if (err) {
473 dev_err(dev, "Notification register failed\n");
474 goto err_service;
475 }
476
477 ppriv->store =
478 dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev);
479 if (!ppriv->store) {
480 err = -ENOMEM;
481 dev_err(dev, "dpaa2_io_store_create() failed\n");
482 goto err_store;
483 }
484
485 ppriv++;
486 }
487 return 0;
488
489 err_store:
490 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
491 err_service:
492 ppriv--;
493 while (ppriv >= priv->ppriv) {
494 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
495 dpaa2_io_store_destroy(ppriv->store);
496 ppriv--;
497 }
498 return err;
499 }
500
dpaa2_dpmai_store_free(struct dpaa2_qdma_priv * priv)501 static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
502 {
503 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
504 int i;
505
506 for (i = 0; i < priv->num_pairs; i++) {
507 dpaa2_io_store_destroy(ppriv->store);
508 ppriv++;
509 }
510 }
511
dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv * priv)512 static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
513 {
514 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
515 struct device *dev = priv->dev;
516 int i;
517
518 for (i = 0; i < priv->num_pairs; i++) {
519 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
520 ppriv++;
521 }
522 }
523
dpaa2_dpdmai_bind(struct dpaa2_qdma_priv * priv)524 static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
525 {
526 struct dpdmai_rx_queue_cfg rx_queue_cfg;
527 struct dpaa2_qdma_priv_per_prio *ppriv;
528 struct device *dev = priv->dev;
529 struct fsl_mc_device *ls_dev;
530 int i, num;
531 int err;
532
533 ls_dev = to_fsl_mc_device(dev);
534 num = priv->num_pairs;
535 ppriv = priv->ppriv;
536 for (i = 0; i < num; i++) {
537 rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
538 DPDMAI_QUEUE_OPT_DEST;
539 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
540 rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
541 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
542 rx_queue_cfg.dest_cfg.priority = ppriv->prio;
543 err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
544 rx_queue_cfg.dest_cfg.priority,
545 &rx_queue_cfg);
546 if (err) {
547 dev_err(dev, "dpdmai_set_rx_queue() failed\n");
548 return err;
549 }
550
551 ppriv++;
552 }
553
554 return 0;
555 }
556
dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv * priv)557 static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
558 {
559 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
560 struct device *dev = priv->dev;
561 struct fsl_mc_device *ls_dev;
562 int err = 0;
563 int i;
564
565 ls_dev = to_fsl_mc_device(dev);
566
567 for (i = 0; i < priv->num_pairs; i++) {
568 ppriv->nctx.qman64 = 0;
569 ppriv->nctx.dpio_id = 0;
570 ppriv++;
571 }
572
573 err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
574 if (err)
575 dev_err(dev, "dpdmai_reset() failed\n");
576
577 return err;
578 }
579
dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan * qchan,struct list_head * head)580 static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
581 struct list_head *head)
582 {
583 struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
584 unsigned long flags;
585
586 list_for_each_entry_safe(comp_tmp, _comp_tmp,
587 head, list) {
588 spin_lock_irqsave(&qchan->queue_lock, flags);
589 list_del(&comp_tmp->list);
590 spin_unlock_irqrestore(&qchan->queue_lock, flags);
591 dma_pool_free(qchan->fd_pool,
592 comp_tmp->fd_virt_addr,
593 comp_tmp->fd_bus_addr);
594 dma_pool_free(qchan->fl_pool,
595 comp_tmp->fl_virt_addr,
596 comp_tmp->fl_bus_addr);
597 dma_pool_free(qchan->sdd_pool,
598 comp_tmp->desc_virt_addr,
599 comp_tmp->desc_bus_addr);
600 kfree(comp_tmp);
601 }
602 }
603
dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine * dpaa2_qdma)604 static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
605 {
606 struct dpaa2_qdma_chan *qchan;
607 int num, i;
608
609 num = dpaa2_qdma->n_chans;
610 for (i = 0; i < num; i++) {
611 qchan = &dpaa2_qdma->chans[i];
612 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
613 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
614 dma_pool_destroy(qchan->fd_pool);
615 dma_pool_destroy(qchan->fl_pool);
616 dma_pool_destroy(qchan->sdd_pool);
617 }
618 }
619
dpaa2_qdma_free_desc(struct virt_dma_desc * vdesc)620 static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
621 {
622 struct dpaa2_qdma_comp *dpaa2_comp;
623 struct dpaa2_qdma_chan *qchan;
624 unsigned long flags;
625
626 dpaa2_comp = to_fsl_qdma_comp(vdesc);
627 qchan = dpaa2_comp->qchan;
628 spin_lock_irqsave(&qchan->queue_lock, flags);
629 list_del(&dpaa2_comp->list);
630 list_add_tail(&dpaa2_comp->list, &qchan->comp_free);
631 spin_unlock_irqrestore(&qchan->queue_lock, flags);
632 }
633
dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine * dpaa2_qdma)634 static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
635 {
636 struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
637 struct dpaa2_qdma_chan *dpaa2_chan;
638 int num = priv->num_pairs;
639 int i;
640
641 INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
642 for (i = 0; i < dpaa2_qdma->n_chans; i++) {
643 dpaa2_chan = &dpaa2_qdma->chans[i];
644 dpaa2_chan->qdma = dpaa2_qdma;
645 dpaa2_chan->fqid = priv->tx_fqid[i % num];
646 dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
647 vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
648 spin_lock_init(&dpaa2_chan->queue_lock);
649 INIT_LIST_HEAD(&dpaa2_chan->comp_used);
650 INIT_LIST_HEAD(&dpaa2_chan->comp_free);
651 }
652 return 0;
653 }
654
dpaa2_qdma_probe(struct fsl_mc_device * dpdmai_dev)655 static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
656 {
657 struct device *dev = &dpdmai_dev->dev;
658 struct dpaa2_qdma_engine *dpaa2_qdma;
659 struct dpaa2_qdma_priv *priv;
660 int err;
661
662 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
663 if (!priv)
664 return -ENOMEM;
665 dev_set_drvdata(dev, priv);
666 priv->dpdmai_dev = dpdmai_dev;
667
668 priv->iommu_domain = iommu_get_domain_for_dev(dev);
669 if (priv->iommu_domain)
670 smmu_disable = false;
671
672 /* obtain a MC portal */
673 err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
674 if (err) {
675 if (err == -ENXIO)
676 err = -EPROBE_DEFER;
677 else
678 dev_err(dev, "MC portal allocation failed\n");
679 goto err_mcportal;
680 }
681
682 /* DPDMAI initialization */
683 err = dpaa2_qdma_setup(dpdmai_dev);
684 if (err) {
685 dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
686 goto err_dpdmai_setup;
687 }
688
689 /* DPIO */
690 err = dpaa2_qdma_dpio_setup(priv);
691 if (err) {
692 dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
693 goto err_dpio_setup;
694 }
695
696 /* DPDMAI binding to DPIO */
697 err = dpaa2_dpdmai_bind(priv);
698 if (err) {
699 dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
700 goto err_bind;
701 }
702
703 /* DPDMAI enable */
704 err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
705 if (err) {
706 dev_err(dev, "dpdmai_enable() faile\n");
707 goto err_enable;
708 }
709
710 dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
711 if (!dpaa2_qdma) {
712 err = -ENOMEM;
713 goto err_eng;
714 }
715
716 priv->dpaa2_qdma = dpaa2_qdma;
717 dpaa2_qdma->priv = priv;
718
719 dpaa2_qdma->desc_allocated = 0;
720 dpaa2_qdma->n_chans = NUM_CH;
721
722 dpaa2_dpdmai_init_channels(dpaa2_qdma);
723
724 if (soc_device_match(soc_fixup_tuning))
725 dpaa2_qdma->qdma_wrtype_fixup = true;
726 else
727 dpaa2_qdma->qdma_wrtype_fixup = false;
728
729 dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
730 dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
731 dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
732
733 dpaa2_qdma->dma_dev.dev = dev;
734 dpaa2_qdma->dma_dev.device_alloc_chan_resources =
735 dpaa2_qdma_alloc_chan_resources;
736 dpaa2_qdma->dma_dev.device_free_chan_resources =
737 dpaa2_qdma_free_chan_resources;
738 dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status;
739 dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
740 dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
741
742 err = dma_async_device_register(&dpaa2_qdma->dma_dev);
743 if (err) {
744 dev_err(dev, "Can't register NXP QDMA engine.\n");
745 goto err_dpaa2_qdma;
746 }
747
748 return 0;
749
750 err_dpaa2_qdma:
751 kfree(dpaa2_qdma);
752 err_eng:
753 dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
754 err_enable:
755 dpaa2_dpdmai_dpio_unbind(priv);
756 err_bind:
757 dpaa2_dpmai_store_free(priv);
758 dpaa2_dpdmai_dpio_free(priv);
759 err_dpio_setup:
760 kfree(priv->ppriv);
761 dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
762 err_dpdmai_setup:
763 fsl_mc_portal_free(priv->mc_io);
764 err_mcportal:
765 kfree(priv);
766 dev_set_drvdata(dev, NULL);
767 return err;
768 }
769
dpaa2_qdma_remove(struct fsl_mc_device * ls_dev)770 static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
771 {
772 struct dpaa2_qdma_engine *dpaa2_qdma;
773 struct dpaa2_qdma_priv *priv;
774 struct device *dev;
775
776 dev = &ls_dev->dev;
777 priv = dev_get_drvdata(dev);
778 dpaa2_qdma = priv->dpaa2_qdma;
779
780 dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
781 dpaa2_dpdmai_dpio_unbind(priv);
782 dpaa2_dpmai_store_free(priv);
783 dpaa2_dpdmai_dpio_free(priv);
784 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
785 fsl_mc_portal_free(priv->mc_io);
786 dev_set_drvdata(dev, NULL);
787 dpaa2_dpdmai_free_channels(dpaa2_qdma);
788
789 dma_async_device_unregister(&dpaa2_qdma->dma_dev);
790 kfree(priv);
791 kfree(dpaa2_qdma);
792
793 return 0;
794 }
795
dpaa2_qdma_shutdown(struct fsl_mc_device * ls_dev)796 static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev)
797 {
798 struct dpaa2_qdma_priv *priv;
799 struct device *dev;
800
801 dev = &ls_dev->dev;
802 priv = dev_get_drvdata(dev);
803
804 dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
805 dpaa2_dpdmai_dpio_unbind(priv);
806 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
807 dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle);
808 }
809
810 static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
811 {
812 .vendor = FSL_MC_VENDOR_FREESCALE,
813 .obj_type = "dpdmai",
814 },
815 { .vendor = 0x0 }
816 };
817
818 static struct fsl_mc_driver dpaa2_qdma_driver = {
819 .driver = {
820 .name = "dpaa2-qdma",
821 .owner = THIS_MODULE,
822 },
823 .probe = dpaa2_qdma_probe,
824 .remove = dpaa2_qdma_remove,
825 .shutdown = dpaa2_qdma_shutdown,
826 .match_id_table = dpaa2_qdma_id_table
827 };
828
dpaa2_qdma_driver_init(void)829 static int __init dpaa2_qdma_driver_init(void)
830 {
831 return fsl_mc_driver_register(&(dpaa2_qdma_driver));
832 }
833 late_initcall(dpaa2_qdma_driver_init);
834
fsl_qdma_exit(void)835 static void __exit fsl_qdma_exit(void)
836 {
837 fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
838 }
839 module_exit(fsl_qdma_exit);
840
841 MODULE_ALIAS("platform:fsl-dpaa2-qdma");
842 MODULE_LICENSE("GPL v2");
843 MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver");
844