1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved.
4 */
5
6 #include <linux/clk.h>
7 #include <linux/delay.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/export.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of.h>
14 #include <linux/of_device.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/sort.h>
18
19 #include <soc/tegra/fuse.h>
20
21 #include "mc.h"
22
23 static const struct of_device_id tegra_mc_of_match[] = {
24 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
25 { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
26 #endif
27 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
28 { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
29 #endif
30 #ifdef CONFIG_ARCH_TEGRA_114_SOC
31 { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
32 #endif
33 #ifdef CONFIG_ARCH_TEGRA_124_SOC
34 { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
35 #endif
36 #ifdef CONFIG_ARCH_TEGRA_132_SOC
37 { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
38 #endif
39 #ifdef CONFIG_ARCH_TEGRA_210_SOC
40 { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
41 #endif
42 #ifdef CONFIG_ARCH_TEGRA_186_SOC
43 { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
44 #endif
45 #ifdef CONFIG_ARCH_TEGRA_194_SOC
46 { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
47 #endif
48 #ifdef CONFIG_ARCH_TEGRA_234_SOC
49 { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
50 #endif
51 { /* sentinel */ }
52 };
53 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
54
tegra_mc_devm_action_put_device(void * data)55 static void tegra_mc_devm_action_put_device(void *data)
56 {
57 struct tegra_mc *mc = data;
58
59 put_device(mc->dev);
60 }
61
62 /**
63 * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
64 * @dev: device pointer for the consumer device
65 *
66 * This function will search for the Memory Controller node in a device-tree
67 * and retrieve the Memory Controller handle.
68 *
69 * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
70 */
devm_tegra_memory_controller_get(struct device * dev)71 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
72 {
73 struct platform_device *pdev;
74 struct device_node *np;
75 struct tegra_mc *mc;
76 int err;
77
78 np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
79 if (!np)
80 return ERR_PTR(-ENOENT);
81
82 pdev = of_find_device_by_node(np);
83 of_node_put(np);
84 if (!pdev)
85 return ERR_PTR(-ENODEV);
86
87 mc = platform_get_drvdata(pdev);
88 if (!mc) {
89 put_device(&pdev->dev);
90 return ERR_PTR(-EPROBE_DEFER);
91 }
92
93 err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
94 if (err)
95 return ERR_PTR(err);
96
97 return mc;
98 }
99 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
100
tegra_mc_probe_device(struct tegra_mc * mc,struct device * dev)101 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
102 {
103 if (mc->soc->ops && mc->soc->ops->probe_device)
104 return mc->soc->ops->probe_device(mc, dev);
105
106 return 0;
107 }
108 EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
109
tegra_mc_block_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)110 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
111 const struct tegra_mc_reset *rst)
112 {
113 unsigned long flags;
114 u32 value;
115
116 spin_lock_irqsave(&mc->lock, flags);
117
118 value = mc_readl(mc, rst->control) | BIT(rst->bit);
119 mc_writel(mc, value, rst->control);
120
121 spin_unlock_irqrestore(&mc->lock, flags);
122
123 return 0;
124 }
125
tegra_mc_dma_idling_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)126 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
127 const struct tegra_mc_reset *rst)
128 {
129 return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
130 }
131
tegra_mc_unblock_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)132 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
133 const struct tegra_mc_reset *rst)
134 {
135 unsigned long flags;
136 u32 value;
137
138 spin_lock_irqsave(&mc->lock, flags);
139
140 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
141 mc_writel(mc, value, rst->control);
142
143 spin_unlock_irqrestore(&mc->lock, flags);
144
145 return 0;
146 }
147
tegra_mc_reset_status_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)148 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
149 const struct tegra_mc_reset *rst)
150 {
151 return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
152 }
153
154 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
155 .block_dma = tegra_mc_block_dma_common,
156 .dma_idling = tegra_mc_dma_idling_common,
157 .unblock_dma = tegra_mc_unblock_dma_common,
158 .reset_status = tegra_mc_reset_status_common,
159 };
160
reset_to_mc(struct reset_controller_dev * rcdev)161 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
162 {
163 return container_of(rcdev, struct tegra_mc, reset);
164 }
165
tegra_mc_reset_find(struct tegra_mc * mc,unsigned long id)166 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
167 unsigned long id)
168 {
169 unsigned int i;
170
171 for (i = 0; i < mc->soc->num_resets; i++)
172 if (mc->soc->resets[i].id == id)
173 return &mc->soc->resets[i];
174
175 return NULL;
176 }
177
tegra_mc_hotreset_assert(struct reset_controller_dev * rcdev,unsigned long id)178 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
179 unsigned long id)
180 {
181 struct tegra_mc *mc = reset_to_mc(rcdev);
182 const struct tegra_mc_reset_ops *rst_ops;
183 const struct tegra_mc_reset *rst;
184 int retries = 500;
185 int err;
186
187 rst = tegra_mc_reset_find(mc, id);
188 if (!rst)
189 return -ENODEV;
190
191 rst_ops = mc->soc->reset_ops;
192 if (!rst_ops)
193 return -ENODEV;
194
195 /* DMA flushing will fail if reset is already asserted */
196 if (rst_ops->reset_status) {
197 /* check whether reset is asserted */
198 if (rst_ops->reset_status(mc, rst))
199 return 0;
200 }
201
202 if (rst_ops->block_dma) {
203 /* block clients DMA requests */
204 err = rst_ops->block_dma(mc, rst);
205 if (err) {
206 dev_err(mc->dev, "failed to block %s DMA: %d\n",
207 rst->name, err);
208 return err;
209 }
210 }
211
212 if (rst_ops->dma_idling) {
213 /* wait for completion of the outstanding DMA requests */
214 while (!rst_ops->dma_idling(mc, rst)) {
215 if (!retries--) {
216 dev_err(mc->dev, "failed to flush %s DMA\n",
217 rst->name);
218 return -EBUSY;
219 }
220
221 usleep_range(10, 100);
222 }
223 }
224
225 if (rst_ops->hotreset_assert) {
226 /* clear clients DMA requests sitting before arbitration */
227 err = rst_ops->hotreset_assert(mc, rst);
228 if (err) {
229 dev_err(mc->dev, "failed to hot reset %s: %d\n",
230 rst->name, err);
231 return err;
232 }
233 }
234
235 return 0;
236 }
237
tegra_mc_hotreset_deassert(struct reset_controller_dev * rcdev,unsigned long id)238 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
239 unsigned long id)
240 {
241 struct tegra_mc *mc = reset_to_mc(rcdev);
242 const struct tegra_mc_reset_ops *rst_ops;
243 const struct tegra_mc_reset *rst;
244 int err;
245
246 rst = tegra_mc_reset_find(mc, id);
247 if (!rst)
248 return -ENODEV;
249
250 rst_ops = mc->soc->reset_ops;
251 if (!rst_ops)
252 return -ENODEV;
253
254 if (rst_ops->hotreset_deassert) {
255 /* take out client from hot reset */
256 err = rst_ops->hotreset_deassert(mc, rst);
257 if (err) {
258 dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
259 rst->name, err);
260 return err;
261 }
262 }
263
264 if (rst_ops->unblock_dma) {
265 /* allow new DMA requests to proceed to arbitration */
266 err = rst_ops->unblock_dma(mc, rst);
267 if (err) {
268 dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
269 rst->name, err);
270 return err;
271 }
272 }
273
274 return 0;
275 }
276
tegra_mc_hotreset_status(struct reset_controller_dev * rcdev,unsigned long id)277 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
278 unsigned long id)
279 {
280 struct tegra_mc *mc = reset_to_mc(rcdev);
281 const struct tegra_mc_reset_ops *rst_ops;
282 const struct tegra_mc_reset *rst;
283
284 rst = tegra_mc_reset_find(mc, id);
285 if (!rst)
286 return -ENODEV;
287
288 rst_ops = mc->soc->reset_ops;
289 if (!rst_ops)
290 return -ENODEV;
291
292 return rst_ops->reset_status(mc, rst);
293 }
294
295 static const struct reset_control_ops tegra_mc_reset_ops = {
296 .assert = tegra_mc_hotreset_assert,
297 .deassert = tegra_mc_hotreset_deassert,
298 .status = tegra_mc_hotreset_status,
299 };
300
tegra_mc_reset_setup(struct tegra_mc * mc)301 static int tegra_mc_reset_setup(struct tegra_mc *mc)
302 {
303 int err;
304
305 mc->reset.ops = &tegra_mc_reset_ops;
306 mc->reset.owner = THIS_MODULE;
307 mc->reset.of_node = mc->dev->of_node;
308 mc->reset.of_reset_n_cells = 1;
309 mc->reset.nr_resets = mc->soc->num_resets;
310
311 err = reset_controller_register(&mc->reset);
312 if (err < 0)
313 return err;
314
315 return 0;
316 }
317
tegra_mc_write_emem_configuration(struct tegra_mc * mc,unsigned long rate)318 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
319 {
320 unsigned int i;
321 struct tegra_mc_timing *timing = NULL;
322
323 for (i = 0; i < mc->num_timings; i++) {
324 if (mc->timings[i].rate == rate) {
325 timing = &mc->timings[i];
326 break;
327 }
328 }
329
330 if (!timing) {
331 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
332 rate);
333 return -EINVAL;
334 }
335
336 for (i = 0; i < mc->soc->num_emem_regs; ++i)
337 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
338
339 return 0;
340 }
341 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
342
tegra_mc_get_emem_device_count(struct tegra_mc * mc)343 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
344 {
345 u8 dram_count;
346
347 dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
348 dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
349 dram_count++;
350
351 return dram_count;
352 }
353 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
354
355 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
356 defined(CONFIG_ARCH_TEGRA_114_SOC) || \
357 defined(CONFIG_ARCH_TEGRA_124_SOC) || \
358 defined(CONFIG_ARCH_TEGRA_132_SOC) || \
359 defined(CONFIG_ARCH_TEGRA_210_SOC)
tegra_mc_setup_latency_allowance(struct tegra_mc * mc)360 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
361 {
362 unsigned long long tick;
363 unsigned int i;
364 u32 value;
365
366 /* compute the number of MC clock cycles per tick */
367 tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
368 do_div(tick, NSEC_PER_SEC);
369
370 value = mc_readl(mc, MC_EMEM_ARB_CFG);
371 value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
372 value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
373 mc_writel(mc, value, MC_EMEM_ARB_CFG);
374
375 /* write latency allowance defaults */
376 for (i = 0; i < mc->soc->num_clients; i++) {
377 const struct tegra_mc_client *client = &mc->soc->clients[i];
378 u32 value;
379
380 value = mc_readl(mc, client->regs.la.reg);
381 value &= ~(client->regs.la.mask << client->regs.la.shift);
382 value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
383 mc_writel(mc, value, client->regs.la.reg);
384 }
385
386 /* latch new values */
387 mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
388
389 return 0;
390 }
391
load_one_timing(struct tegra_mc * mc,struct tegra_mc_timing * timing,struct device_node * node)392 static int load_one_timing(struct tegra_mc *mc,
393 struct tegra_mc_timing *timing,
394 struct device_node *node)
395 {
396 int err;
397 u32 tmp;
398
399 err = of_property_read_u32(node, "clock-frequency", &tmp);
400 if (err) {
401 dev_err(mc->dev,
402 "timing %pOFn: failed to read rate\n", node);
403 return err;
404 }
405
406 timing->rate = tmp;
407 timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
408 sizeof(u32), GFP_KERNEL);
409 if (!timing->emem_data)
410 return -ENOMEM;
411
412 err = of_property_read_u32_array(node, "nvidia,emem-configuration",
413 timing->emem_data,
414 mc->soc->num_emem_regs);
415 if (err) {
416 dev_err(mc->dev,
417 "timing %pOFn: failed to read EMEM configuration\n",
418 node);
419 return err;
420 }
421
422 return 0;
423 }
424
load_timings(struct tegra_mc * mc,struct device_node * node)425 static int load_timings(struct tegra_mc *mc, struct device_node *node)
426 {
427 struct device_node *child;
428 struct tegra_mc_timing *timing;
429 int child_count = of_get_child_count(node);
430 int i = 0, err;
431
432 mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
433 GFP_KERNEL);
434 if (!mc->timings)
435 return -ENOMEM;
436
437 mc->num_timings = child_count;
438
439 for_each_child_of_node(node, child) {
440 timing = &mc->timings[i++];
441
442 err = load_one_timing(mc, timing, child);
443 if (err) {
444 of_node_put(child);
445 return err;
446 }
447 }
448
449 return 0;
450 }
451
tegra_mc_setup_timings(struct tegra_mc * mc)452 static int tegra_mc_setup_timings(struct tegra_mc *mc)
453 {
454 struct device_node *node;
455 u32 ram_code, node_ram_code;
456 int err;
457
458 ram_code = tegra_read_ram_code();
459
460 mc->num_timings = 0;
461
462 for_each_child_of_node(mc->dev->of_node, node) {
463 err = of_property_read_u32(node, "nvidia,ram-code",
464 &node_ram_code);
465 if (err || (node_ram_code != ram_code))
466 continue;
467
468 err = load_timings(mc, node);
469 of_node_put(node);
470 if (err)
471 return err;
472 break;
473 }
474
475 if (mc->num_timings == 0)
476 dev_warn(mc->dev,
477 "no memory timings for RAM code %u registered\n",
478 ram_code);
479
480 return 0;
481 }
482
tegra30_mc_probe(struct tegra_mc * mc)483 int tegra30_mc_probe(struct tegra_mc *mc)
484 {
485 int err;
486
487 mc->clk = devm_clk_get_optional(mc->dev, "mc");
488 if (IS_ERR(mc->clk)) {
489 dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
490 return PTR_ERR(mc->clk);
491 }
492
493 /* ensure that debug features are disabled */
494 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
495
496 err = tegra_mc_setup_latency_allowance(mc);
497 if (err < 0) {
498 dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
499 return err;
500 }
501
502 err = tegra_mc_setup_timings(mc);
503 if (err < 0) {
504 dev_err(mc->dev, "failed to setup timings: %d\n", err);
505 return err;
506 }
507
508 return 0;
509 }
510
511 const struct tegra_mc_ops tegra30_mc_ops = {
512 .probe = tegra30_mc_probe,
513 .handle_irq = tegra30_mc_handle_irq,
514 };
515 #endif
516
mc_global_intstatus_to_channel(const struct tegra_mc * mc,u32 status,unsigned int * mc_channel)517 static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
518 unsigned int *mc_channel)
519 {
520 if ((status & mc->soc->ch_intmask) == 0)
521 return -EINVAL;
522
523 *mc_channel = __ffs((status & mc->soc->ch_intmask) >>
524 mc->soc->global_intstatus_channel_shift);
525
526 return 0;
527 }
528
mc_channel_to_global_intstatus(const struct tegra_mc * mc,unsigned int channel)529 static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
530 unsigned int channel)
531 {
532 return BIT(channel) << mc->soc->global_intstatus_channel_shift;
533 }
534
tegra30_mc_handle_irq(int irq,void * data)535 irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
536 {
537 struct tegra_mc *mc = data;
538 unsigned int bit, channel;
539 unsigned long status;
540
541 if (mc->soc->num_channels) {
542 u32 global_status;
543 int err;
544
545 global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
546 err = mc_global_intstatus_to_channel(mc, global_status, &channel);
547 if (err < 0) {
548 dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
549 global_status);
550 return IRQ_NONE;
551 }
552
553 /* mask all interrupts to avoid flooding */
554 status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
555 } else {
556 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
557 }
558
559 if (!status)
560 return IRQ_NONE;
561
562 for_each_set_bit(bit, &status, 32) {
563 const char *error = tegra_mc_status_names[bit] ?: "unknown";
564 const char *client = "unknown", *desc;
565 const char *direction, *secure;
566 u32 status_reg, addr_reg;
567 u32 intmask = BIT(bit);
568 phys_addr_t addr = 0;
569 #ifdef CONFIG_PHYS_ADDR_T_64BIT
570 u32 addr_hi_reg = 0;
571 #endif
572 unsigned int i;
573 char perm[7];
574 u8 id, type;
575 u32 value;
576
577 switch (intmask) {
578 case MC_INT_DECERR_VPR:
579 status_reg = MC_ERR_VPR_STATUS;
580 addr_reg = MC_ERR_VPR_ADR;
581 break;
582
583 case MC_INT_SECERR_SEC:
584 status_reg = MC_ERR_SEC_STATUS;
585 addr_reg = MC_ERR_SEC_ADR;
586 break;
587
588 case MC_INT_DECERR_MTS:
589 status_reg = MC_ERR_MTS_STATUS;
590 addr_reg = MC_ERR_MTS_ADR;
591 break;
592
593 case MC_INT_DECERR_GENERALIZED_CARVEOUT:
594 status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
595 addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
596 break;
597
598 case MC_INT_DECERR_ROUTE_SANITY:
599 status_reg = MC_ERR_ROUTE_SANITY_STATUS;
600 addr_reg = MC_ERR_ROUTE_SANITY_ADR;
601 break;
602
603 default:
604 status_reg = MC_ERR_STATUS;
605 addr_reg = MC_ERR_ADR;
606
607 #ifdef CONFIG_PHYS_ADDR_T_64BIT
608 if (mc->soc->has_addr_hi_reg)
609 addr_hi_reg = MC_ERR_ADR_HI;
610 #endif
611 break;
612 }
613
614 if (mc->soc->num_channels)
615 value = mc_ch_readl(mc, channel, status_reg);
616 else
617 value = mc_readl(mc, status_reg);
618
619 #ifdef CONFIG_PHYS_ADDR_T_64BIT
620 if (mc->soc->num_address_bits > 32) {
621 if (addr_hi_reg) {
622 if (mc->soc->num_channels)
623 addr = mc_ch_readl(mc, channel, addr_hi_reg);
624 else
625 addr = mc_readl(mc, addr_hi_reg);
626 } else {
627 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
628 MC_ERR_STATUS_ADR_HI_MASK);
629 }
630 addr <<= 32;
631 }
632 #endif
633
634 if (value & MC_ERR_STATUS_RW)
635 direction = "write";
636 else
637 direction = "read";
638
639 if (value & MC_ERR_STATUS_SECURITY)
640 secure = "secure ";
641 else
642 secure = "";
643
644 id = value & mc->soc->client_id_mask;
645
646 for (i = 0; i < mc->soc->num_clients; i++) {
647 if (mc->soc->clients[i].id == id) {
648 client = mc->soc->clients[i].name;
649 break;
650 }
651 }
652
653 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
654 MC_ERR_STATUS_TYPE_SHIFT;
655 desc = tegra_mc_error_names[type];
656
657 switch (value & MC_ERR_STATUS_TYPE_MASK) {
658 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
659 perm[0] = ' ';
660 perm[1] = '[';
661
662 if (value & MC_ERR_STATUS_READABLE)
663 perm[2] = 'R';
664 else
665 perm[2] = '-';
666
667 if (value & MC_ERR_STATUS_WRITABLE)
668 perm[3] = 'W';
669 else
670 perm[3] = '-';
671
672 if (value & MC_ERR_STATUS_NONSECURE)
673 perm[4] = '-';
674 else
675 perm[4] = 'S';
676
677 perm[5] = ']';
678 perm[6] = '\0';
679 break;
680
681 default:
682 perm[0] = '\0';
683 break;
684 }
685
686 if (mc->soc->num_channels)
687 value = mc_ch_readl(mc, channel, addr_reg);
688 else
689 value = mc_readl(mc, addr_reg);
690 addr |= value;
691
692 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
693 client, secure, direction, &addr, error,
694 desc, perm);
695 }
696
697 /* clear interrupts */
698 if (mc->soc->num_channels) {
699 mc_ch_writel(mc, channel, status, MC_INTSTATUS);
700 mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
701 mc_channel_to_global_intstatus(mc, channel),
702 MC_GLOBAL_INTSTATUS);
703 } else {
704 mc_writel(mc, status, MC_INTSTATUS);
705 }
706
707 return IRQ_HANDLED;
708 }
709
710 const char *const tegra_mc_status_names[32] = {
711 [ 1] = "External interrupt",
712 [ 6] = "EMEM address decode error",
713 [ 7] = "GART page fault",
714 [ 8] = "Security violation",
715 [ 9] = "EMEM arbitration error",
716 [10] = "Page fault",
717 [11] = "Invalid APB ASID update",
718 [12] = "VPR violation",
719 [13] = "Secure carveout violation",
720 [16] = "MTS carveout violation",
721 [17] = "Generalized carveout violation",
722 [20] = "Route Sanity error",
723 };
724
725 const char *const tegra_mc_error_names[8] = {
726 [2] = "EMEM decode error",
727 [3] = "TrustZone violation",
728 [4] = "Carveout violation",
729 [6] = "SMMU translation error",
730 };
731
732 /*
733 * Memory Controller (MC) has few Memory Clients that are issuing memory
734 * bandwidth allocation requests to the MC interconnect provider. The MC
735 * provider aggregates the requests and then sends the aggregated request
736 * up to the External Memory Controller (EMC) interconnect provider which
737 * re-configures hardware interface to External Memory (EMEM) in accordance
738 * to the required bandwidth. Each MC interconnect node represents an
739 * individual Memory Client.
740 *
741 * Memory interconnect topology:
742 *
743 * +----+
744 * +--------+ | |
745 * | TEXSRD +--->+ |
746 * +--------+ | |
747 * | | +-----+ +------+
748 * ... | MC +--->+ EMC +--->+ EMEM |
749 * | | +-----+ +------+
750 * +--------+ | |
751 * | DISP.. +--->+ |
752 * +--------+ | |
753 * +----+
754 */
tegra_mc_interconnect_setup(struct tegra_mc * mc)755 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
756 {
757 struct icc_node *node;
758 unsigned int i;
759 int err;
760
761 /* older device-trees don't have interconnect properties */
762 if (!device_property_present(mc->dev, "#interconnect-cells") ||
763 !mc->soc->icc_ops)
764 return 0;
765
766 mc->provider.dev = mc->dev;
767 mc->provider.data = &mc->provider;
768 mc->provider.set = mc->soc->icc_ops->set;
769 mc->provider.aggregate = mc->soc->icc_ops->aggregate;
770 mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
771
772 icc_provider_init(&mc->provider);
773
774 /* create Memory Controller node */
775 node = icc_node_create(TEGRA_ICC_MC);
776 if (IS_ERR(node))
777 return PTR_ERR(node);
778
779 node->name = "Memory Controller";
780 icc_node_add(node, &mc->provider);
781
782 /* link Memory Controller to External Memory Controller */
783 err = icc_link_create(node, TEGRA_ICC_EMC);
784 if (err)
785 goto remove_nodes;
786
787 for (i = 0; i < mc->soc->num_clients; i++) {
788 /* create MC client node */
789 node = icc_node_create(mc->soc->clients[i].id);
790 if (IS_ERR(node)) {
791 err = PTR_ERR(node);
792 goto remove_nodes;
793 }
794
795 node->name = mc->soc->clients[i].name;
796 icc_node_add(node, &mc->provider);
797
798 /* link Memory Client to Memory Controller */
799 err = icc_link_create(node, TEGRA_ICC_MC);
800 if (err)
801 goto remove_nodes;
802 }
803
804 err = icc_provider_register(&mc->provider);
805 if (err)
806 goto remove_nodes;
807
808 return 0;
809
810 remove_nodes:
811 icc_nodes_remove(&mc->provider);
812
813 return err;
814 }
815
tegra_mc_probe(struct platform_device * pdev)816 static int tegra_mc_probe(struct platform_device *pdev)
817 {
818 struct tegra_mc *mc;
819 u64 mask;
820 int err;
821
822 mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
823 if (!mc)
824 return -ENOMEM;
825
826 platform_set_drvdata(pdev, mc);
827 spin_lock_init(&mc->lock);
828 mc->soc = of_device_get_match_data(&pdev->dev);
829 mc->dev = &pdev->dev;
830
831 mask = DMA_BIT_MASK(mc->soc->num_address_bits);
832
833 err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
834 if (err < 0) {
835 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
836 return err;
837 }
838
839 /* length of MC tick in nanoseconds */
840 mc->tick = 30;
841
842 mc->regs = devm_platform_ioremap_resource(pdev, 0);
843 if (IS_ERR(mc->regs))
844 return PTR_ERR(mc->regs);
845
846 mc->debugfs.root = debugfs_create_dir("mc", NULL);
847
848 if (mc->soc->ops && mc->soc->ops->probe) {
849 err = mc->soc->ops->probe(mc);
850 if (err < 0)
851 return err;
852 }
853
854 if (mc->soc->ops && mc->soc->ops->handle_irq) {
855 mc->irq = platform_get_irq(pdev, 0);
856 if (mc->irq < 0)
857 return mc->irq;
858
859 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
860
861 if (mc->soc->num_channels)
862 mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
863 MC_INTMASK);
864 else
865 mc_writel(mc, mc->soc->intmask, MC_INTMASK);
866
867 err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
868 dev_name(&pdev->dev), mc);
869 if (err < 0) {
870 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
871 err);
872 return err;
873 }
874 }
875
876 if (mc->soc->reset_ops) {
877 err = tegra_mc_reset_setup(mc);
878 if (err < 0)
879 dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
880 }
881
882 err = tegra_mc_interconnect_setup(mc);
883 if (err < 0)
884 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
885 err);
886
887 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
888 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
889 if (IS_ERR(mc->smmu)) {
890 dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
891 PTR_ERR(mc->smmu));
892 mc->smmu = NULL;
893 }
894 }
895
896 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
897 mc->gart = tegra_gart_probe(&pdev->dev, mc);
898 if (IS_ERR(mc->gart)) {
899 dev_err(&pdev->dev, "failed to probe GART: %ld\n",
900 PTR_ERR(mc->gart));
901 mc->gart = NULL;
902 }
903 }
904
905 return 0;
906 }
907
tegra_mc_suspend(struct device * dev)908 static int __maybe_unused tegra_mc_suspend(struct device *dev)
909 {
910 struct tegra_mc *mc = dev_get_drvdata(dev);
911
912 if (mc->soc->ops && mc->soc->ops->suspend)
913 return mc->soc->ops->suspend(mc);
914
915 return 0;
916 }
917
tegra_mc_resume(struct device * dev)918 static int __maybe_unused tegra_mc_resume(struct device *dev)
919 {
920 struct tegra_mc *mc = dev_get_drvdata(dev);
921
922 if (mc->soc->ops && mc->soc->ops->resume)
923 return mc->soc->ops->resume(mc);
924
925 return 0;
926 }
927
tegra_mc_sync_state(struct device * dev)928 static void tegra_mc_sync_state(struct device *dev)
929 {
930 struct tegra_mc *mc = dev_get_drvdata(dev);
931
932 /* check whether ICC provider is registered */
933 if (mc->provider.dev == dev)
934 icc_sync_state(dev);
935 }
936
937 static const struct dev_pm_ops tegra_mc_pm_ops = {
938 SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
939 };
940
941 static struct platform_driver tegra_mc_driver = {
942 .driver = {
943 .name = "tegra-mc",
944 .of_match_table = tegra_mc_of_match,
945 .pm = &tegra_mc_pm_ops,
946 .suppress_bind_attrs = true,
947 .sync_state = tegra_mc_sync_state,
948 },
949 .prevent_deferred_probe = true,
950 .probe = tegra_mc_probe,
951 };
952
tegra_mc_init(void)953 static int tegra_mc_init(void)
954 {
955 return platform_driver_register(&tegra_mc_driver);
956 }
957 arch_initcall(tegra_mc_init);
958
959 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
960 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
961 MODULE_LICENSE("GPL v2");
962