1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPU-agnostic ARM page table allocator.
4 * Host-specific functions. The rest is in io-pgtable-arm-common.c.
5 *
6 * Copyright (C) 2014 ARM Limited
7 *
8 * Author: Will Deacon <will.deacon@arm.com>
9 */
10
11 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
12
13 #include <linux/atomic.h>
14 #include <linux/bitops.h>
15 #include <linux/io-pgtable-arm.h>
16 #include <linux/kernel.h>
17 #include <linux/sizes.h>
18 #include <linux/slab.h>
19 #include <linux/types.h>
20 #include <linux/dma-mapping.h>
21
22 #include "iommu-pages.h"
23
24 #include <asm/barrier.h>
25
26 static bool selftest_running = false;
27
arm_lpae_map_exists(void)28 int arm_lpae_map_exists(void)
29 {
30 WARN_ON(!selftest_running);
31 return -EEXIST;
32 }
33
arm_lpae_unmap_empty(void)34 void arm_lpae_unmap_empty(void)
35 {
36 WARN_ON(!selftest_running);
37 }
38
__arm_lpae_dma_addr(void * pages)39 static dma_addr_t __arm_lpae_dma_addr(void *pages)
40 {
41 return (dma_addr_t)virt_to_phys(pages);
42 }
43
__arm_lpae_alloc_pages(size_t size,gfp_t gfp,struct io_pgtable_cfg * cfg,void * cookie)44 void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
45 struct io_pgtable_cfg *cfg,
46 void *cookie)
47 {
48 struct device *dev = cfg->iommu_dev;
49 int order = get_order(size);
50 dma_addr_t dma;
51 void *pages;
52
53 VM_BUG_ON((gfp & __GFP_HIGHMEM));
54
55 if (cfg->alloc)
56 pages = cfg->alloc(cookie, size, gfp);
57 else
58 pages = iommu_alloc_pages_node(dev_to_node(dev), gfp, order);
59
60 if (!pages)
61 return NULL;
62
63 if (!cfg->coherent_walk) {
64 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
65 if (dma_mapping_error(dev, dma))
66 goto out_free;
67 /*
68 * We depend on the IOMMU being able to work with any physical
69 * address directly, so if the DMA layer suggests otherwise by
70 * translating or truncating them, that bodes very badly...
71 */
72 if (dma != virt_to_phys(pages))
73 goto out_unmap;
74 }
75
76 return pages;
77
78 out_unmap:
79 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
80 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
81
82 out_free:
83 if (cfg->free)
84 cfg->free(cookie, pages, size);
85 else
86 iommu_free_pages(pages, order);
87
88 return NULL;
89 }
90
__arm_lpae_free_pages(void * pages,size_t size,struct io_pgtable_cfg * cfg,void * cookie)91 void __arm_lpae_free_pages(void *pages, size_t size,
92 struct io_pgtable_cfg *cfg,
93 void *cookie)
94 {
95 if (!cfg->coherent_walk)
96 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
97 size, DMA_TO_DEVICE);
98
99 if (cfg->free)
100 cfg->free(cookie, pages, size);
101 else
102 iommu_free_pages(pages, get_order(size));
103 }
104
__arm_lpae_sync_pte(arm_lpae_iopte * ptep,int num_entries,struct io_pgtable_cfg * cfg)105 void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
106 struct io_pgtable_cfg *cfg)
107 {
108 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
109 sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
110 }
111
arm_lpae_free_pgtable(struct io_pgtable * iop)112 static void arm_lpae_free_pgtable(struct io_pgtable *iop)
113 {
114 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
115
116 __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
117 kfree(data);
118 }
119
visit_dirty(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)120 static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl,
121 arm_lpae_iopte *ptep, size_t size)
122 {
123 struct io_pgtable_walk_common *walker = walk_data->data;
124 struct iommu_dirty_bitmap *dirty = walker->data;
125
126 if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))
127 return 0;
128
129 if (iopte_writeable_dirty(*ptep)) {
130 iommu_dirty_bitmap_record(dirty, walk_data->addr, size);
131 if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR))
132 iopte_set_writeable_clean(ptep);
133 }
134
135 return 0;
136 }
137
arm_lpae_read_and_clear_dirty(struct io_pgtable_ops * ops,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)138 static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops,
139 unsigned long iova, size_t size,
140 unsigned long flags,
141 struct iommu_dirty_bitmap *dirty)
142 {
143 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
144 struct io_pgtable_cfg *cfg = &data->iop.cfg;
145 struct io_pgtable_walk_common walker = {
146 .data = dirty,
147 };
148 struct io_pgtable_walk_data walk_data = {
149 .iop = &data->iop,
150 .data = &walker,
151 .visit = visit_dirty,
152 .flags = flags,
153 .addr = iova,
154 .end = iova + size,
155 };
156 arm_lpae_iopte *ptep = data->pgd;
157 int lvl = data->start_level;
158
159 if (WARN_ON(!size))
160 return -EINVAL;
161 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1)))
162 return -EINVAL;
163 if (data->iop.fmt != ARM_64_LPAE_S1)
164 return -EINVAL;
165
166 return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl);
167 }
168
169 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)170 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
171 {
172 struct arm_lpae_io_pgtable *data;
173
174 data = kzalloc(sizeof(*data), GFP_KERNEL);
175 if (!data)
176 return NULL;
177
178 if (arm_lpae_init_pgtable_s1(cfg, data))
179 goto out_free_data;
180
181 data->iop.ops.read_and_clear_dirty = arm_lpae_read_and_clear_dirty;
182 /* Looking good; allocate a pgd */
183 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
184 GFP_KERNEL, cfg, cookie);
185 if (!data->pgd)
186 goto out_free_data;
187
188 /* Ensure the empty pgd is visible before any actual TTBR write */
189 wmb();
190
191 /* TTBR */
192 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
193 return &data->iop;
194
195 out_free_data:
196 kfree(data);
197 return NULL;
198 }
199
arm_64_lpae_configure_s1(struct io_pgtable_cfg * cfg)200 static int arm_64_lpae_configure_s1(struct io_pgtable_cfg *cfg)
201 {
202 struct arm_lpae_io_pgtable data = {};
203
204 return arm_lpae_init_pgtable_s1(cfg, &data);
205 }
206
207 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)208 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
209 {
210 struct arm_lpae_io_pgtable *data;
211
212 data = kzalloc(sizeof(*data), GFP_KERNEL);
213 if (!data)
214 return NULL;
215
216 if (arm_lpae_init_pgtable_s2(cfg, data))
217 goto out_free_data;
218
219 data->iop.ops.read_and_clear_dirty = arm_lpae_read_and_clear_dirty;
220 /* Allocate pgd pages */
221 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
222 GFP_KERNEL, cfg, cookie);
223 if (!data->pgd)
224 goto out_free_data;
225
226 /* Ensure the empty pgd is visible before any actual TTBR write */
227 wmb();
228
229 /* VTTBR */
230 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
231 return &data->iop;
232
233 out_free_data:
234 kfree(data);
235 return NULL;
236 }
237
arm_64_lpae_configure_s2(struct io_pgtable_cfg * cfg)238 static int arm_64_lpae_configure_s2(struct io_pgtable_cfg *cfg)
239 {
240 struct arm_lpae_io_pgtable data = {};
241
242 return arm_lpae_init_pgtable_s2(cfg, &data);
243 }
244
245 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)246 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
247 {
248 if (cfg->ias > 32 || cfg->oas > 40)
249 return NULL;
250
251 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
252 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
253 }
254
255 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)256 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
257 {
258 if (cfg->ias > 40 || cfg->oas > 40)
259 return NULL;
260
261 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
262 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
263 }
264
265 static struct io_pgtable *
arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg * cfg,void * cookie)266 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
267 {
268 struct arm_lpae_io_pgtable *data;
269
270 /* No quirks for Mali (hopefully) */
271 if (cfg->quirks)
272 return NULL;
273
274 if (cfg->ias > 48 || cfg->oas > 40)
275 return NULL;
276
277 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
278
279 data = kzalloc(sizeof(*data), GFP_KERNEL);
280 if (!data)
281 return NULL;
282
283 if (arm_lpae_init_pgtable(cfg, data))
284 return NULL;
285
286 data->iop.ops.read_and_clear_dirty = arm_lpae_read_and_clear_dirty;
287 /* Mali seems to need a full 4-level table regardless of IAS */
288 if (data->start_level > 0) {
289 data->start_level = 0;
290 data->pgd_bits = 0;
291 }
292 /*
293 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
294 * best we can do is mimic the out-of-tree driver and hope that the
295 * "implementation-defined caching policy" is good enough. Similarly,
296 * we'll use it for the sake of a valid attribute for our 'device'
297 * index, although callers should never request that in practice.
298 */
299 cfg->arm_mali_lpae_cfg.memattr =
300 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
301 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
302 (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
303 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
304 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
305 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
306
307 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
308 cfg, cookie);
309 if (!data->pgd)
310 goto out_free_data;
311
312 /* Ensure the empty pgd is visible before TRANSTAB can be written */
313 wmb();
314
315 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
316 ARM_MALI_LPAE_TTBR_READ_INNER |
317 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
318 if (cfg->coherent_walk)
319 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;
320
321 return &data->iop;
322
323 out_free_data:
324 kfree(data);
325 return NULL;
326 }
327
328 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
329 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
330 .alloc = arm_64_lpae_alloc_pgtable_s1,
331 .free = arm_lpae_free_pgtable,
332 .configure = arm_64_lpae_configure_s1,
333 };
334
335 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
336 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
337 .alloc = arm_64_lpae_alloc_pgtable_s2,
338 .free = arm_lpae_free_pgtable,
339 .configure = arm_64_lpae_configure_s2,
340 };
341
342 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
343 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
344 .alloc = arm_32_lpae_alloc_pgtable_s1,
345 .free = arm_lpae_free_pgtable,
346 };
347
348 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
349 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
350 .alloc = arm_32_lpae_alloc_pgtable_s2,
351 .free = arm_lpae_free_pgtable,
352 };
353
354 struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
355 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
356 .alloc = arm_mali_lpae_alloc_pgtable,
357 .free = arm_lpae_free_pgtable,
358 };
359
360 #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
361
362 static struct io_pgtable_cfg *cfg_cookie __initdata;
363 static struct io_pgtable_ops *cur_ops;
364
dummy_tlb_flush_all(void * cookie)365 static void __init dummy_tlb_flush_all(void *cookie)
366 {
367 WARN_ON(cookie != cfg_cookie);
368 }
369
370 static unsigned long skip_addr = 0xFFFF;
arm_lpae_selftest_validate(phys_addr_t addr,size_t size,struct io_pgtable_walk_common * data,void * wd)371 static void arm_lpae_selftest_validate(phys_addr_t addr, size_t size,
372 struct io_pgtable_walk_common *data,
373 void *wd)
374 {
375 struct arm_lpae_io_pgtable_walk_data *arm_wd = data->data;
376 unsigned long *iova = (unsigned long *)(arm_wd->cookie);
377 arm_lpae_iopte *ptep = wd;
378
379 /* PASS */
380 if (*iova == addr)
381 *iova = *iova + size;
382
383 WARN_ON(skip_addr == addr);
384 WARN_ON(!(*ptep));
385 *ptep = 0;
386 }
387
dummy_tlb_flush(unsigned long iova,size_t size,size_t granule,void * cookie)388 static void __init dummy_tlb_flush(unsigned long iova, size_t size,
389 size_t granule, void *cookie)
390
391 {
392 unsigned long iova_cookie = iova;
393 struct arm_lpae_io_pgtable_walk_data wd = {
394 .cookie = &iova_cookie,
395 };
396 struct io_pgtable_walk_common walk_data = {
397 .visit_leaf = arm_lpae_selftest_validate,
398 .data = &wd,
399 };
400
401 if (cur_ops && (cfg_cookie->quirks & IO_PGTABLE_QUIRK_UNMAP_INVAL)) {
402 /* Not straight forward to propagate failures, so WARN_ON is noisy enough. */
403 cur_ops->pgtable_walk(cur_ops, iova, size, &walk_data);
404 }
405
406 WARN_ON(cookie != cfg_cookie);
407 WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
408 }
409
dummy_tlb_add_page(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)410 static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
411 unsigned long iova, size_t granule,
412 void *cookie)
413 {
414 WARN_ON(cookie != cfg_cookie);
415 WARN_ON(!(granule & cfg_cookie->pgsize_bitmap));
416 }
417
418 static const struct iommu_flush_ops dummy_tlb_ops __initconst = {
419 .tlb_flush_all = dummy_tlb_flush_all,
420 .tlb_flush_walk = dummy_tlb_flush,
421 .tlb_add_page = dummy_tlb_add_page,
422 };
423
arm_lpae_dump_ops(struct io_pgtable_ops * ops)424 static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
425 {
426 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
427 struct io_pgtable_cfg *cfg = &data->iop.cfg;
428
429 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",
430 cfg->pgsize_bitmap, cfg->ias);
431 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
432 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
433 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd);
434 }
435
436 #define __FAIL(ops, i) ({ \
437 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \
438 arm_lpae_dump_ops(ops); \
439 selftest_running = false; \
440 -EFAULT; \
441 })
442
arm_lpae_run_tests(struct io_pgtable_cfg * cfg)443 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
444 {
445 static const enum io_pgtable_fmt fmts[] __initconst = {
446 ARM_64_LPAE_S1,
447 ARM_64_LPAE_S2,
448 };
449
450 int i, j;
451 unsigned long iova, iova_cookie;
452 size_t size, mapped;
453 struct io_pgtable_ops *ops;
454 struct arm_lpae_io_pgtable_walk_data arm_wd;
455 struct io_pgtable_walk_common common_wd;
456 int ret;
457
458 selftest_running = true;
459
460 for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
461 cfg_cookie = cfg;
462 cfg->quirks = 0;
463 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
464 if (!ops) {
465 pr_err("selftest: failed to allocate io pgtable ops\n");
466 return -ENOMEM;
467 }
468
469 /*
470 * Initial sanity checks.
471 * Empty page tables shouldn't provide any translations.
472 */
473 if (ops->iova_to_phys(ops, 42))
474 return __FAIL(ops, i);
475
476 if (ops->iova_to_phys(ops, SZ_1G + 42))
477 return __FAIL(ops, i);
478
479 if (ops->iova_to_phys(ops, SZ_2G + 42))
480 return __FAIL(ops, i);
481
482 /*
483 * Distinct mappings of different granule sizes.
484 */
485 iova = 0;
486 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
487 size = 1UL << j;
488
489 if (ops->map_pages(ops, iova, iova, size, 1,
490 IOMMU_READ | IOMMU_WRITE |
491 IOMMU_NOEXEC | IOMMU_CACHE,
492 GFP_KERNEL, &mapped))
493 return __FAIL(ops, i);
494
495 /* Overlapping mappings */
496 if (!ops->map_pages(ops, iova, iova + size, size, 1,
497 IOMMU_READ | IOMMU_NOEXEC,
498 GFP_KERNEL, &mapped))
499 return __FAIL(ops, i);
500
501 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
502 return __FAIL(ops, i);
503
504 iova += SZ_1G;
505 }
506
507 /* Partial unmap */
508 size = 1UL << __ffs(cfg->pgsize_bitmap);
509 if (ops->unmap_pages(ops, SZ_1G + size, size, 1, NULL) != size)
510 return __FAIL(ops, i);
511
512 /* Remap of partial unmap */
513 if (ops->map_pages(ops, SZ_1G + size, size, size, 1,
514 IOMMU_READ, GFP_KERNEL, &mapped))
515 return __FAIL(ops, i);
516
517 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
518 return __FAIL(ops, i);
519
520 /* Full unmap */
521 iova = 0;
522 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
523 size = 1UL << j;
524
525 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
526 return __FAIL(ops, i);
527
528 if (ops->iova_to_phys(ops, iova + 42))
529 return __FAIL(ops, i);
530
531 /* Remap full block */
532 if (ops->map_pages(ops, iova, iova, size, 1,
533 IOMMU_WRITE, GFP_KERNEL, &mapped))
534 return __FAIL(ops, i);
535
536 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
537 return __FAIL(ops, i);
538
539 iova += SZ_1G;
540 }
541
542 free_io_pgtable_ops(ops);
543
544 /* Test: IO_PGTABLE_QUIRK_UNMAP_INVAL */
545 cfg->quirks = IO_PGTABLE_QUIRK_UNMAP_INVAL;
546 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
547 cur_ops = ops;
548 if (!ops) {
549 pr_err("selftest: failed to allocate io pgtable ops with IO_PGTABLE_QUIRK_UNMAP_INVAL\n");
550 return -ENOMEM;
551 }
552
553 common_wd.visit_leaf = arm_lpae_selftest_validate;
554 common_wd.data = &arm_wd;
555 arm_wd.cookie = &iova_cookie;
556
557 /*
558 * Map with leaf size => unmap with leaf size
559 * Then walk the table to check the pages
560 */
561 size = 1UL << __ffs(cfg->pgsize_bitmap);
562 iova = size * 3; /* Arbitrary aligned address. */
563 if (ops->map_pages(ops, iova, iova, size, 1,
564 IOMMU_READ | IOMMU_WRITE |
565 IOMMU_NOEXEC | IOMMU_CACHE,
566 GFP_KERNEL, &mapped))
567 return __FAIL(ops, i);
568
569 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
570 return __FAIL(ops, i);
571
572 iova_cookie = iova;
573 ret = ops->pgtable_walk(ops, iova, size, &common_wd);
574 if (ret || (iova_cookie != iova + size))
575 return __FAIL(ops, i);
576
577 /*
578 * Map with leaf size => partial unmap with leaf size
579 * Then walk the table to check the pages
580 */
581 if (ops->map_pages(ops, iova, iova, size, 42,
582 IOMMU_READ | IOMMU_WRITE |
583 IOMMU_NOEXEC | IOMMU_CACHE,
584 GFP_KERNEL, &mapped))
585 return __FAIL(ops, i);
586
587 if (ops->unmap_pages(ops, iova + 41 * size, size, 1, NULL) != size)
588 return __FAIL(ops, i);
589
590 iova_cookie = iova + 41 * size;
591 ret = ops->pgtable_walk(ops, iova_cookie, size, &common_wd);
592 if (ret || (iova_cookie != (iova + 42 * size)))
593 return __FAIL(ops, i);
594
595 if (ops->unmap_pages(ops, iova, size, 41, NULL) != 41 * size)
596 return __FAIL(ops, i);
597
598 iova_cookie = iova;
599 ret = ops->pgtable_walk(ops, iova, size * 41, &common_wd);
600 if (ret || (iova_cookie != (iova + 41 * size)))
601 return __FAIL(ops, i);
602 /*
603 * Distinct mappings of different granule sizes.
604 */
605 iova = 0;
606 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
607 size = 1UL << j;
608
609 if (ops->map_pages(ops, iova, iova, size, 1,
610 IOMMU_READ | IOMMU_WRITE |
611 IOMMU_NOEXEC | IOMMU_CACHE,
612 GFP_KERNEL, &mapped))
613 return __FAIL(ops, i);
614
615 /* Overlapping mappings */
616 if (!ops->map_pages(ops, iova, iova + size, size, 1,
617 IOMMU_READ | IOMMU_NOEXEC,
618 GFP_KERNEL, &mapped))
619 return __FAIL(ops, i);
620
621 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
622 return __FAIL(ops, i);
623
624 iova += SZ_1G;
625 }
626
627 /* Partial unmap (split blk) */
628 size = 1UL << __ffs(cfg->pgsize_bitmap);
629 iova = SZ_1G + size;
630 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
631 return __FAIL(ops, i);
632
633 /* Now we have table instead of block, with missing entry let's see */
634 iova_cookie = iova;
635 ret = ops->pgtable_walk(ops, iova, size, &common_wd);
636 if (ret || (iova_cookie != (iova + size)))
637 return __FAIL(ops, i);
638
639 if (ops->iova_to_phys(ops, iova))
640 return __FAIL(ops, i);
641
642 /*
643 * Let's replace with a block again.
644 * We expect the freed table will be called in tlb_flush_walk()
645 * that's how we can track the unmapped pages.
646 */
647 size = 1ULL << __ffs(cfg->pgsize_bitmap & ~(1UL << __ffs(cfg->pgsize_bitmap)));
648 /* Already unmapped shouldn't walk it again! */
649 skip_addr = SZ_1G + size;
650 iova = SZ_1G;
651 iova_cookie = iova;
652 if (ops->map_pages(ops, iova, iova, size, 1,
653 IOMMU_READ, GFP_KERNEL, &mapped))
654 return __FAIL(ops, i);
655 skip_addr = 0XFFFF;
656
657 /* Let's break the block to table again, this time at the start. */
658 size = 1UL << __ffs(cfg->pgsize_bitmap);
659
660 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
661 return __FAIL(ops, i);
662
663 /* Now we have table instead of block, with missing entry let's see */
664 iova_cookie = iova;
665 ret = ops->pgtable_walk(ops, iova_cookie, size, &common_wd);
666 if (ret || (iova_cookie != (iova + size)))
667 return __FAIL(ops, i);
668
669 if (ops->iova_to_phys(ops, iova))
670 return __FAIL(ops, i);
671
672 /* Let's unmap the whole table at once. */
673 size = 1ULL << __ffs(cfg->pgsize_bitmap & ~(1UL << __ffs(cfg->pgsize_bitmap)));
674 skip_addr = iova;
675 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
676 return __FAIL(ops, i);
677 skip_addr = 0xFFFF;
678 cur_ops = NULL;
679 free_io_pgtable_ops(ops);
680 }
681
682 selftest_running = false;
683 return 0;
684 }
685
arm_lpae_do_selftests(void)686 static int __init arm_lpae_do_selftests(void)
687 {
688 static const unsigned long pgsize[] __initconst = {
689 SZ_4K | SZ_2M | SZ_1G,
690 SZ_16K | SZ_32M,
691 SZ_64K | SZ_512M,
692 };
693
694 static const unsigned int ias[] __initconst = {
695 32, 36, 40, 42, 44, 48,
696 };
697
698 int i, j, pass = 0, fail = 0;
699 struct device dev;
700 struct io_pgtable_cfg cfg = {
701 .tlb = &dummy_tlb_ops,
702 .oas = 48,
703 .coherent_walk = true,
704 .iommu_dev = &dev,
705 };
706
707 /* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */
708 set_dev_node(&dev, NUMA_NO_NODE);
709
710 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) {
711 for (j = 0; j < ARRAY_SIZE(ias); ++j) {
712 cfg.pgsize_bitmap = pgsize[i];
713 cfg.ias = ias[j];
714 pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n",
715 pgsize[i], ias[j]);
716 if (arm_lpae_run_tests(&cfg))
717 fail++;
718 else
719 pass++;
720 }
721 }
722
723 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail);
724 return fail ? -EFAULT : 0;
725 }
726 subsys_initcall(arm_lpae_do_selftests);
727 #endif
728