1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for Graphics Address Relocation Table on Tegra20
4 *
5 * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Author: Hiroshi DOYU <hdoyu@nvidia.com>
8 */
9
10 #define dev_fmt(fmt) "gart: " fmt
11
12 #include <linux/io.h>
13 #include <linux/iommu.h>
14 #include <linux/moduleparam.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/vmalloc.h>
19
20 #include <soc/tegra/mc.h>
21
22 #define GART_REG_BASE 0x24
23 #define GART_CONFIG (0x24 - GART_REG_BASE)
24 #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE)
25 #define GART_ENTRY_DATA (0x2c - GART_REG_BASE)
26
27 #define GART_ENTRY_PHYS_ADDR_VALID BIT(31)
28
29 #define GART_PAGE_SHIFT 12
30 #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
31 #define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT)
32
33 /* bitmap of the page sizes currently supported */
34 #define GART_IOMMU_PGSIZES (GART_PAGE_SIZE)
35
36 struct gart_device {
37 void __iomem *regs;
38 u32 *savedata;
39 unsigned long iovmm_base; /* offset to vmm_area start */
40 unsigned long iovmm_end; /* offset to vmm_area end */
41 spinlock_t pte_lock; /* for pagetable */
42 spinlock_t dom_lock; /* for active domain */
43 unsigned int active_devices; /* number of active devices */
44 struct iommu_domain *active_domain; /* current active domain */
45 struct iommu_device iommu; /* IOMMU Core handle */
46 struct device *dev;
47 };
48
49 static struct gart_device *gart_handle; /* unique for a system */
50
51 static bool gart_debug;
52
53 /*
54 * Any interaction between any block on PPSB and a block on APB or AHB
55 * must have these read-back to ensure the APB/AHB bus transaction is
56 * complete before initiating activity on the PPSB block.
57 */
58 #define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG)
59
60 #define for_each_gart_pte(gart, iova) \
61 for (iova = gart->iovmm_base; \
62 iova < gart->iovmm_end; \
63 iova += GART_PAGE_SIZE)
64
gart_set_pte(struct gart_device * gart,unsigned long iova,unsigned long pte)65 static inline void gart_set_pte(struct gart_device *gart,
66 unsigned long iova, unsigned long pte)
67 {
68 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
69 writel_relaxed(pte, gart->regs + GART_ENTRY_DATA);
70 }
71
gart_read_pte(struct gart_device * gart,unsigned long iova)72 static inline unsigned long gart_read_pte(struct gart_device *gart,
73 unsigned long iova)
74 {
75 unsigned long pte;
76
77 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
78 pte = readl_relaxed(gart->regs + GART_ENTRY_DATA);
79
80 return pte;
81 }
82
do_gart_setup(struct gart_device * gart,const u32 * data)83 static void do_gart_setup(struct gart_device *gart, const u32 *data)
84 {
85 unsigned long iova;
86
87 for_each_gart_pte(gart, iova)
88 gart_set_pte(gart, iova, data ? *(data++) : 0);
89
90 writel_relaxed(1, gart->regs + GART_CONFIG);
91 FLUSH_GART_REGS(gart);
92 }
93
gart_iova_range_invalid(struct gart_device * gart,unsigned long iova,size_t bytes)94 static inline bool gart_iova_range_invalid(struct gart_device *gart,
95 unsigned long iova, size_t bytes)
96 {
97 return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE ||
98 iova + bytes > gart->iovmm_end);
99 }
100
gart_pte_valid(struct gart_device * gart,unsigned long iova)101 static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova)
102 {
103 return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID);
104 }
105
gart_iommu_attach_dev(struct iommu_domain * domain,struct device * dev)106 static int gart_iommu_attach_dev(struct iommu_domain *domain,
107 struct device *dev)
108 {
109 struct gart_device *gart = gart_handle;
110 int ret = 0;
111
112 spin_lock(&gart->dom_lock);
113
114 if (gart->active_domain && gart->active_domain != domain) {
115 ret = -EBUSY;
116 } else if (dev->archdata.iommu != domain) {
117 dev->archdata.iommu = domain;
118 gart->active_domain = domain;
119 gart->active_devices++;
120 }
121
122 spin_unlock(&gart->dom_lock);
123
124 return ret;
125 }
126
gart_iommu_detach_dev(struct iommu_domain * domain,struct device * dev)127 static void gart_iommu_detach_dev(struct iommu_domain *domain,
128 struct device *dev)
129 {
130 struct gart_device *gart = gart_handle;
131
132 spin_lock(&gart->dom_lock);
133
134 if (dev->archdata.iommu == domain) {
135 dev->archdata.iommu = NULL;
136
137 if (--gart->active_devices == 0)
138 gart->active_domain = NULL;
139 }
140
141 spin_unlock(&gart->dom_lock);
142 }
143
gart_iommu_domain_alloc(unsigned type)144 static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
145 {
146 struct iommu_domain *domain;
147
148 if (type != IOMMU_DOMAIN_UNMANAGED)
149 return NULL;
150
151 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
152 if (domain) {
153 domain->geometry.aperture_start = gart_handle->iovmm_base;
154 domain->geometry.aperture_end = gart_handle->iovmm_end - 1;
155 domain->geometry.force_aperture = true;
156 }
157
158 return domain;
159 }
160
gart_iommu_domain_free(struct iommu_domain * domain)161 static void gart_iommu_domain_free(struct iommu_domain *domain)
162 {
163 WARN_ON(gart_handle->active_domain == domain);
164 kfree(domain);
165 }
166
__gart_iommu_map(struct gart_device * gart,unsigned long iova,unsigned long pa)167 static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova,
168 unsigned long pa)
169 {
170 if (unlikely(gart_debug && gart_pte_valid(gart, iova))) {
171 dev_err(gart->dev, "Page entry is in-use\n");
172 return -EINVAL;
173 }
174
175 gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa);
176
177 return 0;
178 }
179
gart_iommu_map(struct iommu_domain * domain,unsigned long iova,phys_addr_t pa,size_t bytes,int prot)180 static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
181 phys_addr_t pa, size_t bytes, int prot)
182 {
183 struct gart_device *gart = gart_handle;
184 int ret;
185
186 if (gart_iova_range_invalid(gart, iova, bytes))
187 return -EINVAL;
188
189 spin_lock(&gart->pte_lock);
190 ret = __gart_iommu_map(gart, iova, (unsigned long)pa);
191 spin_unlock(&gart->pte_lock);
192
193 return ret;
194 }
195
__gart_iommu_unmap(struct gart_device * gart,unsigned long iova)196 static inline int __gart_iommu_unmap(struct gart_device *gart,
197 unsigned long iova)
198 {
199 if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) {
200 dev_err(gart->dev, "Page entry is invalid\n");
201 return -EINVAL;
202 }
203
204 gart_set_pte(gart, iova, 0);
205
206 return 0;
207 }
208
gart_iommu_unmap(struct iommu_domain * domain,unsigned long iova,size_t bytes,struct iommu_iotlb_gather * gather)209 static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
210 size_t bytes, struct iommu_iotlb_gather *gather)
211 {
212 struct gart_device *gart = gart_handle;
213 int err;
214
215 if (gart_iova_range_invalid(gart, iova, bytes))
216 return 0;
217
218 spin_lock(&gart->pte_lock);
219 err = __gart_iommu_unmap(gart, iova);
220 spin_unlock(&gart->pte_lock);
221
222 return err ? 0 : bytes;
223 }
224
gart_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)225 static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
226 dma_addr_t iova)
227 {
228 struct gart_device *gart = gart_handle;
229 unsigned long pte;
230
231 if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE))
232 return -EINVAL;
233
234 spin_lock(&gart->pte_lock);
235 pte = gart_read_pte(gart, iova);
236 spin_unlock(&gart->pte_lock);
237
238 return pte & GART_PAGE_MASK;
239 }
240
gart_iommu_capable(enum iommu_cap cap)241 static bool gart_iommu_capable(enum iommu_cap cap)
242 {
243 return false;
244 }
245
gart_iommu_add_device(struct device * dev)246 static int gart_iommu_add_device(struct device *dev)
247 {
248 struct iommu_group *group;
249
250 if (!dev->iommu_fwspec)
251 return -ENODEV;
252
253 group = iommu_group_get_for_dev(dev);
254 if (IS_ERR(group))
255 return PTR_ERR(group);
256
257 iommu_group_put(group);
258
259 iommu_device_link(&gart_handle->iommu, dev);
260
261 return 0;
262 }
263
gart_iommu_remove_device(struct device * dev)264 static void gart_iommu_remove_device(struct device *dev)
265 {
266 iommu_group_remove_device(dev);
267 iommu_device_unlink(&gart_handle->iommu, dev);
268 }
269
gart_iommu_of_xlate(struct device * dev,struct of_phandle_args * args)270 static int gart_iommu_of_xlate(struct device *dev,
271 struct of_phandle_args *args)
272 {
273 return 0;
274 }
275
gart_iommu_sync_map(struct iommu_domain * domain)276 static void gart_iommu_sync_map(struct iommu_domain *domain)
277 {
278 FLUSH_GART_REGS(gart_handle);
279 }
280
gart_iommu_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)281 static void gart_iommu_sync(struct iommu_domain *domain,
282 struct iommu_iotlb_gather *gather)
283 {
284 gart_iommu_sync_map(domain);
285 }
286
287 static const struct iommu_ops gart_iommu_ops = {
288 .capable = gart_iommu_capable,
289 .domain_alloc = gart_iommu_domain_alloc,
290 .domain_free = gart_iommu_domain_free,
291 .attach_dev = gart_iommu_attach_dev,
292 .detach_dev = gart_iommu_detach_dev,
293 .add_device = gart_iommu_add_device,
294 .remove_device = gart_iommu_remove_device,
295 .device_group = generic_device_group,
296 .map = gart_iommu_map,
297 .unmap = gart_iommu_unmap,
298 .iova_to_phys = gart_iommu_iova_to_phys,
299 .pgsize_bitmap = GART_IOMMU_PGSIZES,
300 .of_xlate = gart_iommu_of_xlate,
301 .iotlb_sync_map = gart_iommu_sync_map,
302 .iotlb_sync = gart_iommu_sync,
303 };
304
tegra_gart_suspend(struct gart_device * gart)305 int tegra_gart_suspend(struct gart_device *gart)
306 {
307 u32 *data = gart->savedata;
308 unsigned long iova;
309
310 /*
311 * All GART users shall be suspended at this point. Disable
312 * address translation to trap all GART accesses as invalid
313 * memory accesses.
314 */
315 writel_relaxed(0, gart->regs + GART_CONFIG);
316 FLUSH_GART_REGS(gart);
317
318 for_each_gart_pte(gart, iova)
319 *(data++) = gart_read_pte(gart, iova);
320
321 return 0;
322 }
323
tegra_gart_resume(struct gart_device * gart)324 int tegra_gart_resume(struct gart_device *gart)
325 {
326 do_gart_setup(gart, gart->savedata);
327
328 return 0;
329 }
330
tegra_gart_probe(struct device * dev,struct tegra_mc * mc)331 struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
332 {
333 struct gart_device *gart;
334 struct resource *res;
335 int err;
336
337 BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
338
339 /* the GART memory aperture is required */
340 res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1);
341 if (!res) {
342 dev_err(dev, "Memory aperture resource unavailable\n");
343 return ERR_PTR(-ENXIO);
344 }
345
346 gart = kzalloc(sizeof(*gart), GFP_KERNEL);
347 if (!gart)
348 return ERR_PTR(-ENOMEM);
349
350 gart_handle = gart;
351
352 gart->dev = dev;
353 gart->regs = mc->regs + GART_REG_BASE;
354 gart->iovmm_base = res->start;
355 gart->iovmm_end = res->end + 1;
356 spin_lock_init(&gart->pte_lock);
357 spin_lock_init(&gart->dom_lock);
358
359 do_gart_setup(gart, NULL);
360
361 err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
362 if (err)
363 goto free_gart;
364
365 iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
366 iommu_device_set_fwnode(&gart->iommu, dev->fwnode);
367
368 err = iommu_device_register(&gart->iommu);
369 if (err)
370 goto remove_sysfs;
371
372 gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE *
373 sizeof(u32));
374 if (!gart->savedata) {
375 err = -ENOMEM;
376 goto unregister_iommu;
377 }
378
379 return gart;
380
381 unregister_iommu:
382 iommu_device_unregister(&gart->iommu);
383 remove_sysfs:
384 iommu_device_sysfs_remove(&gart->iommu);
385 free_gart:
386 kfree(gart);
387
388 return ERR_PTR(err);
389 }
390
391 module_param(gart_debug, bool, 0644);
392 MODULE_PARM_DESC(gart_debug, "Enable GART debugging");
393