1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Tegra host1x driver
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 */
7
8 #include <linux/clk.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/io.h>
11 #include <linux/list.h>
12 #include <linux/module.h>
13 #include <linux/of_device.h>
14 #include <linux/of.h>
15 #include <linux/slab.h>
16
17 #define CREATE_TRACE_POINTS
18 #include <trace/events/host1x.h>
19 #undef CREATE_TRACE_POINTS
20
21 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
22 #include <asm/dma-iommu.h>
23 #endif
24
25 #include "bus.h"
26 #include "channel.h"
27 #include "debug.h"
28 #include "dev.h"
29 #include "intr.h"
30
31 #include "hw/host1x01.h"
32 #include "hw/host1x02.h"
33 #include "hw/host1x04.h"
34 #include "hw/host1x05.h"
35 #include "hw/host1x06.h"
36 #include "hw/host1x07.h"
37
host1x_hypervisor_writel(struct host1x * host1x,u32 v,u32 r)38 void host1x_hypervisor_writel(struct host1x *host1x, u32 v, u32 r)
39 {
40 writel(v, host1x->hv_regs + r);
41 }
42
host1x_hypervisor_readl(struct host1x * host1x,u32 r)43 u32 host1x_hypervisor_readl(struct host1x *host1x, u32 r)
44 {
45 return readl(host1x->hv_regs + r);
46 }
47
host1x_sync_writel(struct host1x * host1x,u32 v,u32 r)48 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r)
49 {
50 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset;
51
52 writel(v, sync_regs + r);
53 }
54
host1x_sync_readl(struct host1x * host1x,u32 r)55 u32 host1x_sync_readl(struct host1x *host1x, u32 r)
56 {
57 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset;
58
59 return readl(sync_regs + r);
60 }
61
host1x_ch_writel(struct host1x_channel * ch,u32 v,u32 r)62 void host1x_ch_writel(struct host1x_channel *ch, u32 v, u32 r)
63 {
64 writel(v, ch->regs + r);
65 }
66
host1x_ch_readl(struct host1x_channel * ch,u32 r)67 u32 host1x_ch_readl(struct host1x_channel *ch, u32 r)
68 {
69 return readl(ch->regs + r);
70 }
71
72 static const struct host1x_info host1x01_info = {
73 .nb_channels = 8,
74 .nb_pts = 32,
75 .nb_mlocks = 16,
76 .nb_bases = 8,
77 .init = host1x01_init,
78 .sync_offset = 0x3000,
79 .dma_mask = DMA_BIT_MASK(32),
80 .has_wide_gather = false,
81 .has_hypervisor = false,
82 .num_sid_entries = 0,
83 .sid_table = NULL,
84 .reserve_vblank_syncpts = true,
85 };
86
87 static const struct host1x_info host1x02_info = {
88 .nb_channels = 9,
89 .nb_pts = 32,
90 .nb_mlocks = 16,
91 .nb_bases = 12,
92 .init = host1x02_init,
93 .sync_offset = 0x3000,
94 .dma_mask = DMA_BIT_MASK(32),
95 .has_wide_gather = false,
96 .has_hypervisor = false,
97 .num_sid_entries = 0,
98 .sid_table = NULL,
99 .reserve_vblank_syncpts = true,
100 };
101
102 static const struct host1x_info host1x04_info = {
103 .nb_channels = 12,
104 .nb_pts = 192,
105 .nb_mlocks = 16,
106 .nb_bases = 64,
107 .init = host1x04_init,
108 .sync_offset = 0x2100,
109 .dma_mask = DMA_BIT_MASK(34),
110 .has_wide_gather = false,
111 .has_hypervisor = false,
112 .num_sid_entries = 0,
113 .sid_table = NULL,
114 .reserve_vblank_syncpts = false,
115 };
116
117 static const struct host1x_info host1x05_info = {
118 .nb_channels = 14,
119 .nb_pts = 192,
120 .nb_mlocks = 16,
121 .nb_bases = 64,
122 .init = host1x05_init,
123 .sync_offset = 0x2100,
124 .dma_mask = DMA_BIT_MASK(34),
125 .has_wide_gather = false,
126 .has_hypervisor = false,
127 .num_sid_entries = 0,
128 .sid_table = NULL,
129 .reserve_vblank_syncpts = false,
130 };
131
132 static const struct host1x_sid_entry tegra186_sid_table[] = {
133 {
134 /* VIC */
135 .base = 0x1af0,
136 .offset = 0x30,
137 .limit = 0x34
138 },
139 };
140
141 static const struct host1x_info host1x06_info = {
142 .nb_channels = 63,
143 .nb_pts = 576,
144 .nb_mlocks = 24,
145 .nb_bases = 16,
146 .init = host1x06_init,
147 .sync_offset = 0x0,
148 .dma_mask = DMA_BIT_MASK(40),
149 .has_wide_gather = true,
150 .has_hypervisor = true,
151 .num_sid_entries = ARRAY_SIZE(tegra186_sid_table),
152 .sid_table = tegra186_sid_table,
153 .reserve_vblank_syncpts = false,
154 };
155
156 static const struct host1x_sid_entry tegra194_sid_table[] = {
157 {
158 /* VIC */
159 .base = 0x1af0,
160 .offset = 0x30,
161 .limit = 0x34
162 },
163 };
164
165 static const struct host1x_info host1x07_info = {
166 .nb_channels = 63,
167 .nb_pts = 704,
168 .nb_mlocks = 32,
169 .nb_bases = 0,
170 .init = host1x07_init,
171 .sync_offset = 0x0,
172 .dma_mask = DMA_BIT_MASK(40),
173 .has_wide_gather = true,
174 .has_hypervisor = true,
175 .num_sid_entries = ARRAY_SIZE(tegra194_sid_table),
176 .sid_table = tegra194_sid_table,
177 .reserve_vblank_syncpts = false,
178 };
179
180 static const struct of_device_id host1x_of_match[] = {
181 { .compatible = "nvidia,tegra194-host1x", .data = &host1x07_info, },
182 { .compatible = "nvidia,tegra186-host1x", .data = &host1x06_info, },
183 { .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, },
184 { .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, },
185 { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, },
186 { .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, },
187 { .compatible = "nvidia,tegra20-host1x", .data = &host1x01_info, },
188 { },
189 };
190 MODULE_DEVICE_TABLE(of, host1x_of_match);
191
host1x_setup_sid_table(struct host1x * host)192 static void host1x_setup_sid_table(struct host1x *host)
193 {
194 const struct host1x_info *info = host->info;
195 unsigned int i;
196
197 for (i = 0; i < info->num_sid_entries; i++) {
198 const struct host1x_sid_entry *entry = &info->sid_table[i];
199
200 host1x_hypervisor_writel(host, entry->offset, entry->base);
201 host1x_hypervisor_writel(host, entry->limit, entry->base + 4);
202 }
203 }
204
host1x_wants_iommu(struct host1x * host1x)205 static bool host1x_wants_iommu(struct host1x *host1x)
206 {
207 /* Our IOMMU usage policy doesn't currently play well with GART */
208 if (of_machine_is_compatible("nvidia,tegra20"))
209 return false;
210
211 /*
212 * If we support addressing a maximum of 32 bits of physical memory
213 * and if the host1x firewall is enabled, there's no need to enable
214 * IOMMU support. This can happen for example on Tegra20, Tegra30
215 * and Tegra114.
216 *
217 * Tegra124 and later can address up to 34 bits of physical memory and
218 * many platforms come equipped with more than 2 GiB of system memory,
219 * which requires crossing the 4 GiB boundary. But there's a catch: on
220 * SoCs before Tegra186 (i.e. Tegra124 and Tegra210), the host1x can
221 * only address up to 32 bits of memory in GATHER opcodes, which means
222 * that command buffers need to either be in the first 2 GiB of system
223 * memory (which could quickly lead to memory exhaustion), or command
224 * buffers need to be treated differently from other buffers (which is
225 * not possible with the current ABI).
226 *
227 * A third option is to use the IOMMU in these cases to make sure all
228 * buffers will be mapped into a 32-bit IOVA space that host1x can
229 * address. This allows all of the system memory to be used and works
230 * within the limitations of the host1x on these SoCs.
231 *
232 * In summary, default to enable IOMMU on Tegra124 and later. For any
233 * of the earlier SoCs, only use the IOMMU for additional safety when
234 * the host1x firewall is disabled.
235 */
236 if (host1x->info->dma_mask <= DMA_BIT_MASK(32)) {
237 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL))
238 return false;
239 }
240
241 return true;
242 }
243
host1x_iommu_attach(struct host1x * host)244 static struct iommu_domain *host1x_iommu_attach(struct host1x *host)
245 {
246 struct iommu_domain *domain = iommu_get_domain_for_dev(host->dev);
247 int err;
248
249 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
250 if (host->dev->archdata.mapping) {
251 struct dma_iommu_mapping *mapping =
252 to_dma_iommu_mapping(host->dev);
253 arm_iommu_detach_device(host->dev);
254 arm_iommu_release_mapping(mapping);
255
256 domain = iommu_get_domain_for_dev(host->dev);
257 }
258 #endif
259
260 /*
261 * We may not always want to enable IOMMU support (for example if the
262 * host1x firewall is already enabled and we don't support addressing
263 * more than 32 bits of physical memory), so check for that first.
264 *
265 * Similarly, if host1x is already attached to an IOMMU (via the DMA
266 * API), don't try to attach again.
267 */
268 if (!host1x_wants_iommu(host) || domain)
269 return domain;
270
271 host->group = iommu_group_get(host->dev);
272 if (host->group) {
273 struct iommu_domain_geometry *geometry;
274 dma_addr_t start, end;
275 unsigned long order;
276
277 err = iova_cache_get();
278 if (err < 0)
279 goto put_group;
280
281 host->domain = iommu_domain_alloc(&platform_bus_type);
282 if (!host->domain) {
283 err = -ENOMEM;
284 goto put_cache;
285 }
286
287 err = iommu_attach_group(host->domain, host->group);
288 if (err) {
289 if (err == -ENODEV)
290 err = 0;
291
292 goto free_domain;
293 }
294
295 geometry = &host->domain->geometry;
296 start = geometry->aperture_start & host->info->dma_mask;
297 end = geometry->aperture_end & host->info->dma_mask;
298
299 order = __ffs(host->domain->pgsize_bitmap);
300 init_iova_domain(&host->iova, 1UL << order, start >> order);
301 host->iova_end = end;
302
303 domain = host->domain;
304 }
305
306 return domain;
307
308 free_domain:
309 iommu_domain_free(host->domain);
310 host->domain = NULL;
311 put_cache:
312 iova_cache_put();
313 put_group:
314 iommu_group_put(host->group);
315 host->group = NULL;
316
317 return ERR_PTR(err);
318 }
319
host1x_iommu_init(struct host1x * host)320 static int host1x_iommu_init(struct host1x *host)
321 {
322 u64 mask = host->info->dma_mask;
323 struct iommu_domain *domain;
324 int err;
325
326 domain = host1x_iommu_attach(host);
327 if (IS_ERR(domain)) {
328 err = PTR_ERR(domain);
329 dev_err(host->dev, "failed to attach to IOMMU: %d\n", err);
330 return err;
331 }
332
333 /*
334 * If we're not behind an IOMMU make sure we don't get push buffers
335 * that are allocated outside of the range addressable by the GATHER
336 * opcode.
337 *
338 * Newer generations of Tegra (Tegra186 and later) support a wide
339 * variant of the GATHER opcode that allows addressing more bits.
340 */
341 if (!domain && !host->info->has_wide_gather)
342 mask = DMA_BIT_MASK(32);
343
344 err = dma_coerce_mask_and_coherent(host->dev, mask);
345 if (err < 0) {
346 dev_err(host->dev, "failed to set DMA mask: %d\n", err);
347 return err;
348 }
349
350 return 0;
351 }
352
host1x_iommu_exit(struct host1x * host)353 static void host1x_iommu_exit(struct host1x *host)
354 {
355 if (host->domain) {
356 put_iova_domain(&host->iova);
357 iommu_detach_group(host->domain, host->group);
358
359 iommu_domain_free(host->domain);
360 host->domain = NULL;
361
362 iova_cache_put();
363
364 iommu_group_put(host->group);
365 host->group = NULL;
366 }
367 }
368
host1x_probe(struct platform_device * pdev)369 static int host1x_probe(struct platform_device *pdev)
370 {
371 struct host1x *host;
372 struct resource *regs, *hv_regs = NULL;
373 int syncpt_irq;
374 int err;
375
376 host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL);
377 if (!host)
378 return -ENOMEM;
379
380 host->info = of_device_get_match_data(&pdev->dev);
381
382 if (host->info->has_hypervisor) {
383 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vm");
384 if (!regs) {
385 dev_err(&pdev->dev, "failed to get vm registers\n");
386 return -ENXIO;
387 }
388
389 hv_regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
390 "hypervisor");
391 if (!hv_regs) {
392 dev_err(&pdev->dev,
393 "failed to get hypervisor registers\n");
394 return -ENXIO;
395 }
396 } else {
397 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
398 if (!regs) {
399 dev_err(&pdev->dev, "failed to get registers\n");
400 return -ENXIO;
401 }
402 }
403
404 syncpt_irq = platform_get_irq(pdev, 0);
405 if (syncpt_irq < 0)
406 return syncpt_irq;
407
408 mutex_init(&host->devices_lock);
409 INIT_LIST_HEAD(&host->devices);
410 INIT_LIST_HEAD(&host->list);
411 host->dev = &pdev->dev;
412
413 /* set common host1x device data */
414 platform_set_drvdata(pdev, host);
415
416 host->regs = devm_ioremap_resource(&pdev->dev, regs);
417 if (IS_ERR(host->regs))
418 return PTR_ERR(host->regs);
419
420 if (host->info->has_hypervisor) {
421 host->hv_regs = devm_ioremap_resource(&pdev->dev, hv_regs);
422 if (IS_ERR(host->hv_regs))
423 return PTR_ERR(host->hv_regs);
424 }
425
426 host->dev->dma_parms = &host->dma_parms;
427 dma_set_max_seg_size(host->dev, UINT_MAX);
428
429 if (host->info->init) {
430 err = host->info->init(host);
431 if (err)
432 return err;
433 }
434
435 host->clk = devm_clk_get(&pdev->dev, NULL);
436 if (IS_ERR(host->clk)) {
437 err = PTR_ERR(host->clk);
438
439 if (err != -EPROBE_DEFER)
440 dev_err(&pdev->dev, "failed to get clock: %d\n", err);
441
442 return err;
443 }
444
445 host->rst = devm_reset_control_get(&pdev->dev, "host1x");
446 if (IS_ERR(host->rst)) {
447 err = PTR_ERR(host->rst);
448 dev_err(&pdev->dev, "failed to get reset: %d\n", err);
449 return err;
450 }
451
452 err = host1x_iommu_init(host);
453 if (err < 0) {
454 dev_err(&pdev->dev, "failed to setup IOMMU: %d\n", err);
455 return err;
456 }
457
458 err = host1x_channel_list_init(&host->channel_list,
459 host->info->nb_channels);
460 if (err) {
461 dev_err(&pdev->dev, "failed to initialize channel list\n");
462 goto iommu_exit;
463 }
464
465 err = clk_prepare_enable(host->clk);
466 if (err < 0) {
467 dev_err(&pdev->dev, "failed to enable clock\n");
468 goto free_channels;
469 }
470
471 err = reset_control_deassert(host->rst);
472 if (err < 0) {
473 dev_err(&pdev->dev, "failed to deassert reset: %d\n", err);
474 goto unprepare_disable;
475 }
476
477 err = host1x_syncpt_init(host);
478 if (err) {
479 dev_err(&pdev->dev, "failed to initialize syncpts\n");
480 goto reset_assert;
481 }
482
483 err = host1x_intr_init(host, syncpt_irq);
484 if (err) {
485 dev_err(&pdev->dev, "failed to initialize interrupts\n");
486 goto deinit_syncpt;
487 }
488
489 host1x_debug_init(host);
490
491 if (host->info->has_hypervisor)
492 host1x_setup_sid_table(host);
493
494 err = host1x_register(host);
495 if (err < 0)
496 goto deinit_debugfs;
497
498 err = devm_of_platform_populate(&pdev->dev);
499 if (err < 0)
500 goto unregister;
501
502 return 0;
503
504 unregister:
505 host1x_unregister(host);
506 deinit_debugfs:
507 host1x_debug_deinit(host);
508 host1x_intr_deinit(host);
509 deinit_syncpt:
510 host1x_syncpt_deinit(host);
511 reset_assert:
512 reset_control_assert(host->rst);
513 unprepare_disable:
514 clk_disable_unprepare(host->clk);
515 free_channels:
516 host1x_channel_list_free(&host->channel_list);
517 iommu_exit:
518 host1x_iommu_exit(host);
519
520 return err;
521 }
522
host1x_remove(struct platform_device * pdev)523 static int host1x_remove(struct platform_device *pdev)
524 {
525 struct host1x *host = platform_get_drvdata(pdev);
526
527 host1x_unregister(host);
528 host1x_debug_deinit(host);
529 host1x_intr_deinit(host);
530 host1x_syncpt_deinit(host);
531 reset_control_assert(host->rst);
532 clk_disable_unprepare(host->clk);
533 host1x_channel_list_free(&host->channel_list);
534 host1x_iommu_exit(host);
535
536 return 0;
537 }
538
539 static struct platform_driver tegra_host1x_driver = {
540 .driver = {
541 .name = "tegra-host1x",
542 .of_match_table = host1x_of_match,
543 },
544 .probe = host1x_probe,
545 .remove = host1x_remove,
546 };
547
548 static struct platform_driver * const drivers[] = {
549 &tegra_host1x_driver,
550 &tegra_mipi_driver,
551 };
552
tegra_host1x_init(void)553 static int __init tegra_host1x_init(void)
554 {
555 int err;
556
557 err = bus_register(&host1x_bus_type);
558 if (err < 0)
559 return err;
560
561 err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
562 if (err < 0)
563 bus_unregister(&host1x_bus_type);
564
565 return err;
566 }
567 module_init(tegra_host1x_init);
568
tegra_host1x_exit(void)569 static void __exit tegra_host1x_exit(void)
570 {
571 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
572 bus_unregister(&host1x_bus_type);
573 }
574 module_exit(tegra_host1x_exit);
575
576 /**
577 * host1x_get_dma_mask() - query the supported DMA mask for host1x
578 * @host1x: host1x instance
579 *
580 * Note that this returns the supported DMA mask for host1x, which can be
581 * different from the applicable DMA mask under certain circumstances.
582 */
host1x_get_dma_mask(struct host1x * host1x)583 u64 host1x_get_dma_mask(struct host1x *host1x)
584 {
585 return host1x->info->dma_mask;
586 }
587 EXPORT_SYMBOL(host1x_get_dma_mask);
588
589 MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>");
590 MODULE_AUTHOR("Terje Bergstrom <tbergstrom@nvidia.com>");
591 MODULE_DESCRIPTION("Host1x driver for Tegra products");
592 MODULE_LICENSE("GPL");
593