Lines Matching +full:data +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * CPU-agnostic ARM page table allocator.
10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
14 #include <linux/io-pgtable.h>
19 #include <linux/dma-mapping.h>
23 #include "io-pgtable-arm.h"
41 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
45 (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
47 (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
54 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
58 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
60 /* Calculate the block/page mapping size at level l for pagetable in d. */
90 /* Stage-1 PTE */
96 /* Stage-2 PTE */
153 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) in iopte_leaf()
160 struct arm_lpae_io_pgtable *data) in paddr_to_iopte() argument
165 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; in paddr_to_iopte()
169 struct arm_lpae_io_pgtable *data) in iopte_to_paddr() argument
173 if (ARM_LPAE_GRANULE(data) < SZ_64K) in iopte_to_paddr()
176 /* Rotate the packed high-order bits back to the top */ in iopte_to_paddr()
177 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); in iopte_to_paddr()
187 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, in __arm_lpae_alloc_pages() argument
190 struct device *dev = cfg->iommu_dev; in __arm_lpae_alloc_pages()
191 int order = get_order(size); in __arm_lpae_alloc_pages()
203 if (!cfg->coherent_walk) { in __arm_lpae_alloc_pages()
204 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); in __arm_lpae_alloc_pages()
220 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); in __arm_lpae_alloc_pages()
226 static void __arm_lpae_free_pages(void *pages, size_t size, in __arm_lpae_free_pages() argument
229 if (!cfg->coherent_walk) in __arm_lpae_free_pages()
230 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), in __arm_lpae_free_pages()
231 size, DMA_TO_DEVICE); in __arm_lpae_free_pages()
232 free_pages((unsigned long)pages, get_order(size)); in __arm_lpae_free_pages()
238 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte()
247 if (!cfg->coherent_walk) in __arm_lpae_set_pte()
251 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
253 unsigned long iova, size_t size, int lvl,
256 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, in __arm_lpae_init_pte() argument
262 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) in __arm_lpae_init_pte()
267 pte |= paddr_to_iopte(paddr, data); in __arm_lpae_init_pte()
269 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); in __arm_lpae_init_pte()
272 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_init_pte() argument
279 if (iopte_leaf(pte, lvl, data->iop.fmt)) { in arm_lpae_init_pte()
282 return -EEXIST; in arm_lpae_init_pte()
289 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in arm_lpae_init_pte()
291 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_init_pte()
292 if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) { in arm_lpae_init_pte()
294 return -EINVAL; in arm_lpae_init_pte()
298 __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); in arm_lpae_init_pte()
305 struct arm_lpae_io_pgtable *data) in arm_lpae_install_table() argument
308 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_install_table()
310 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE; in arm_lpae_install_table()
311 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_install_table()
323 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) in arm_lpae_install_table()
334 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, in __arm_lpae_map() argument
335 phys_addr_t paddr, size_t size, arm_lpae_iopte prot, in __arm_lpae_map() argument
339 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); in __arm_lpae_map()
340 size_t tblsz = ARM_LPAE_GRANULE(data); in __arm_lpae_map()
341 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_map()
344 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_map()
347 if (size == block_size) in __arm_lpae_map()
348 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); in __arm_lpae_map()
351 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) in __arm_lpae_map()
352 return -EINVAL; in __arm_lpae_map()
359 return -ENOMEM; in __arm_lpae_map()
361 pte = arm_lpae_install_table(cptep, ptep, 0, data); in __arm_lpae_map()
364 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { in __arm_lpae_map()
368 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { in __arm_lpae_map()
369 cptep = iopte_deref(pte, data); in __arm_lpae_map()
373 return -EEXIST; in __arm_lpae_map()
377 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); in __arm_lpae_map()
380 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_prot_to_pte() argument
385 if (data->iop.fmt == ARM_64_LPAE_S1 || in arm_lpae_prot_to_pte()
386 data->iop.fmt == ARM_32_LPAE_S1) { in arm_lpae_prot_to_pte()
402 * having stage-1-like attributes but stage-2-like permissions. in arm_lpae_prot_to_pte()
404 if (data->iop.fmt == ARM_64_LPAE_S2 || in arm_lpae_prot_to_pte()
405 data->iop.fmt == ARM_32_LPAE_S2) { in arm_lpae_prot_to_pte()
427 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) in arm_lpae_prot_to_pte()
435 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_prot_to_pte()
438 if (data->iop.fmt != ARM_MALI_LPAE) in arm_lpae_prot_to_pte()
445 phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp) in arm_lpae_map() argument
447 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_map() local
448 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_map()
449 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_map()
450 int ret, lvl = data->start_level; in arm_lpae_map()
452 long iaext = (s64)iova >> cfg->ias; in arm_lpae_map()
458 if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) in arm_lpae_map()
459 return -EINVAL; in arm_lpae_map()
461 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_map()
463 if (WARN_ON(iaext || paddr >> cfg->oas)) in arm_lpae_map()
464 return -ERANGE; in arm_lpae_map()
466 prot = arm_lpae_prot_to_pte(data, iommu_prot); in arm_lpae_map()
467 ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, gfp); in arm_lpae_map()
477 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, in __arm_lpae_free_pgtable() argument
483 if (lvl == data->start_level) in __arm_lpae_free_pgtable()
484 table_size = ARM_LPAE_PGD_SIZE(data); in __arm_lpae_free_pgtable()
486 table_size = ARM_LPAE_GRANULE(data); in __arm_lpae_free_pgtable()
491 if (lvl == ARM_LPAE_MAX_LEVELS - 1) in __arm_lpae_free_pgtable()
499 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) in __arm_lpae_free_pgtable()
502 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); in __arm_lpae_free_pgtable()
505 __arm_lpae_free_pages(start, table_size, &data->iop.cfg); in __arm_lpae_free_pgtable()
510 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); in arm_lpae_free_pgtable() local
512 __arm_lpae_free_pgtable(data, data->start_level, data->pgd); in arm_lpae_free_pgtable()
513 kfree(data); in arm_lpae_free_pgtable()
516 static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, in arm_lpae_split_blk_unmap() argument
518 unsigned long iova, size_t size, in arm_lpae_split_blk_unmap() argument
522 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_split_blk_unmap()
525 size_t tablesz = ARM_LPAE_GRANULE(data); in arm_lpae_split_blk_unmap()
526 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in arm_lpae_split_blk_unmap()
527 int i, unmap_idx = -1; in arm_lpae_split_blk_unmap()
536 if (size == split_sz) in arm_lpae_split_blk_unmap()
537 unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_split_blk_unmap()
539 blk_paddr = iopte_to_paddr(blk_pte, data); in arm_lpae_split_blk_unmap()
547 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); in arm_lpae_split_blk_unmap()
550 pte = arm_lpae_install_table(tablep, ptep, blk_pte, data); in arm_lpae_split_blk_unmap()
558 if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE) in arm_lpae_split_blk_unmap()
561 tablep = iopte_deref(pte, data); in arm_lpae_split_blk_unmap()
563 io_pgtable_tlb_add_page(&data->iop, gather, iova, size); in arm_lpae_split_blk_unmap()
564 return size; in arm_lpae_split_blk_unmap()
567 return __arm_lpae_unmap(data, gather, iova, size, lvl, tablep); in arm_lpae_split_blk_unmap()
570 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, in __arm_lpae_unmap() argument
572 unsigned long iova, size_t size, int lvl, in __arm_lpae_unmap() argument
576 struct io_pgtable *iop = &data->iop; in __arm_lpae_unmap()
582 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_unmap()
587 /* If the size matches this level, we're in the right place */ in __arm_lpae_unmap()
588 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { in __arm_lpae_unmap()
589 __arm_lpae_set_pte(ptep, 0, &iop->cfg); in __arm_lpae_unmap()
591 if (!iopte_leaf(pte, lvl, iop->fmt)) { in __arm_lpae_unmap()
593 io_pgtable_tlb_flush_walk(iop, iova, size, in __arm_lpae_unmap()
594 ARM_LPAE_GRANULE(data)); in __arm_lpae_unmap()
595 ptep = iopte_deref(pte, data); in __arm_lpae_unmap()
596 __arm_lpae_free_pgtable(data, lvl + 1, ptep); in __arm_lpae_unmap()
597 } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) { in __arm_lpae_unmap()
605 io_pgtable_tlb_add_page(iop, gather, iova, size); in __arm_lpae_unmap()
608 return size; in __arm_lpae_unmap()
609 } else if (iopte_leaf(pte, lvl, iop->fmt)) { in __arm_lpae_unmap()
614 return arm_lpae_split_blk_unmap(data, gather, iova, size, pte, in __arm_lpae_unmap()
619 ptep = iopte_deref(pte, data); in __arm_lpae_unmap()
620 return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep); in __arm_lpae_unmap()
624 size_t size, struct iommu_iotlb_gather *gather) in arm_lpae_unmap() argument
626 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_unmap() local
627 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_unmap()
628 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_unmap()
629 long iaext = (s64)iova >> cfg->ias; in arm_lpae_unmap()
631 if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) in arm_lpae_unmap()
634 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_unmap()
639 return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep); in arm_lpae_unmap()
645 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_iova_to_phys() local
646 arm_lpae_iopte pte, *ptep = data->pgd; in arm_lpae_iova_to_phys()
647 int lvl = data->start_level; in arm_lpae_iova_to_phys()
655 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_iova_to_phys()
663 if (iopte_leaf(pte, lvl, data->iop.fmt)) in arm_lpae_iova_to_phys()
667 ptep = iopte_deref(pte, data); in arm_lpae_iova_to_phys()
674 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); in arm_lpae_iova_to_phys()
675 return iopte_to_paddr(pte, data) | iova; in arm_lpae_iova_to_phys()
686 * the CPU page size if possible, otherwise prefer smaller sizes. in arm_lpae_restrict_pgsizes()
690 if (cfg->pgsize_bitmap & PAGE_SIZE) in arm_lpae_restrict_pgsizes()
692 else if (cfg->pgsize_bitmap & ~PAGE_MASK) in arm_lpae_restrict_pgsizes()
693 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); in arm_lpae_restrict_pgsizes()
694 else if (cfg->pgsize_bitmap & PAGE_MASK) in arm_lpae_restrict_pgsizes()
695 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); in arm_lpae_restrict_pgsizes()
709 if (cfg->oas > 48) in arm_lpae_restrict_pgsizes()
716 cfg->pgsize_bitmap &= page_sizes; in arm_lpae_restrict_pgsizes()
717 cfg->ias = min(cfg->ias, max_addr_bits); in arm_lpae_restrict_pgsizes()
718 cfg->oas = min(cfg->oas, max_addr_bits); in arm_lpae_restrict_pgsizes()
724 struct arm_lpae_io_pgtable *data; in arm_lpae_alloc_pgtable() local
729 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) in arm_lpae_alloc_pgtable()
732 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
735 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
738 data = kmalloc(sizeof(*data), GFP_KERNEL); in arm_lpae_alloc_pgtable()
739 if (!data) in arm_lpae_alloc_pgtable()
742 pg_shift = __ffs(cfg->pgsize_bitmap); in arm_lpae_alloc_pgtable()
743 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte)); in arm_lpae_alloc_pgtable()
745 va_bits = cfg->ias - pg_shift; in arm_lpae_alloc_pgtable()
746 levels = DIV_ROUND_UP(va_bits, data->bits_per_level); in arm_lpae_alloc_pgtable()
747 data->start_level = ARM_LPAE_MAX_LEVELS - levels; in arm_lpae_alloc_pgtable()
749 /* Calculate the actual size of our pgd (without concatenation) */ in arm_lpae_alloc_pgtable()
750 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); in arm_lpae_alloc_pgtable()
752 data->iop.ops = (struct io_pgtable_ops) { in arm_lpae_alloc_pgtable()
758 return data; in arm_lpae_alloc_pgtable()
765 struct arm_lpae_io_pgtable *data; in arm_64_lpae_alloc_pgtable_s1() local
766 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; in arm_64_lpae_alloc_pgtable_s1()
769 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | in arm_64_lpae_alloc_pgtable_s1()
774 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s1()
775 if (!data) in arm_64_lpae_alloc_pgtable_s1()
779 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s1()
780 tcr->sh = ARM_LPAE_TCR_SH_IS; in arm_64_lpae_alloc_pgtable_s1()
781 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s1()
782 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s1()
784 tcr->sh = ARM_LPAE_TCR_SH_OS; in arm_64_lpae_alloc_pgtable_s1()
785 tcr->irgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s1()
786 tcr->orgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s1()
789 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; in arm_64_lpae_alloc_pgtable_s1()
790 switch (ARM_LPAE_GRANULE(data)) { in arm_64_lpae_alloc_pgtable_s1()
792 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; in arm_64_lpae_alloc_pgtable_s1()
795 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; in arm_64_lpae_alloc_pgtable_s1()
798 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; in arm_64_lpae_alloc_pgtable_s1()
802 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s1()
804 tcr->ips = ARM_LPAE_TCR_PS_32_BIT; in arm_64_lpae_alloc_pgtable_s1()
807 tcr->ips = ARM_LPAE_TCR_PS_36_BIT; in arm_64_lpae_alloc_pgtable_s1()
810 tcr->ips = ARM_LPAE_TCR_PS_40_BIT; in arm_64_lpae_alloc_pgtable_s1()
813 tcr->ips = ARM_LPAE_TCR_PS_42_BIT; in arm_64_lpae_alloc_pgtable_s1()
816 tcr->ips = ARM_LPAE_TCR_PS_44_BIT; in arm_64_lpae_alloc_pgtable_s1()
819 tcr->ips = ARM_LPAE_TCR_PS_48_BIT; in arm_64_lpae_alloc_pgtable_s1()
822 tcr->ips = ARM_LPAE_TCR_PS_52_BIT; in arm_64_lpae_alloc_pgtable_s1()
828 tcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s1()
840 cfg->arm_lpae_s1_cfg.mair = reg; in arm_64_lpae_alloc_pgtable_s1()
843 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), in arm_64_lpae_alloc_pgtable_s1()
845 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s1()
852 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s1()
853 return &data->iop; in arm_64_lpae_alloc_pgtable_s1()
856 kfree(data); in arm_64_lpae_alloc_pgtable_s1()
864 struct arm_lpae_io_pgtable *data; in arm_64_lpae_alloc_pgtable_s2() local
865 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; in arm_64_lpae_alloc_pgtable_s2()
868 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT)) in arm_64_lpae_alloc_pgtable_s2()
871 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s2()
872 if (!data) in arm_64_lpae_alloc_pgtable_s2()
877 * the depth of the stage-2 walk. in arm_64_lpae_alloc_pgtable_s2()
879 if (data->start_level == 0) { in arm_64_lpae_alloc_pgtable_s2()
882 pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); in arm_64_lpae_alloc_pgtable_s2()
884 data->pgd_bits += data->bits_per_level; in arm_64_lpae_alloc_pgtable_s2()
885 data->start_level++; in arm_64_lpae_alloc_pgtable_s2()
890 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s2()
891 vtcr->sh = ARM_LPAE_TCR_SH_IS; in arm_64_lpae_alloc_pgtable_s2()
892 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s2()
893 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s2()
895 vtcr->sh = ARM_LPAE_TCR_SH_OS; in arm_64_lpae_alloc_pgtable_s2()
896 vtcr->irgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s2()
897 vtcr->orgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s2()
900 sl = data->start_level; in arm_64_lpae_alloc_pgtable_s2()
902 switch (ARM_LPAE_GRANULE(data)) { in arm_64_lpae_alloc_pgtable_s2()
904 vtcr->tg = ARM_LPAE_TCR_TG0_4K; in arm_64_lpae_alloc_pgtable_s2()
905 sl++; /* SL0 format is different for 4K granule size */ in arm_64_lpae_alloc_pgtable_s2()
908 vtcr->tg = ARM_LPAE_TCR_TG0_16K; in arm_64_lpae_alloc_pgtable_s2()
911 vtcr->tg = ARM_LPAE_TCR_TG0_64K; in arm_64_lpae_alloc_pgtable_s2()
915 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s2()
917 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; in arm_64_lpae_alloc_pgtable_s2()
920 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; in arm_64_lpae_alloc_pgtable_s2()
923 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; in arm_64_lpae_alloc_pgtable_s2()
926 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; in arm_64_lpae_alloc_pgtable_s2()
929 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; in arm_64_lpae_alloc_pgtable_s2()
932 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; in arm_64_lpae_alloc_pgtable_s2()
935 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; in arm_64_lpae_alloc_pgtable_s2()
941 vtcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s2()
942 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; in arm_64_lpae_alloc_pgtable_s2()
945 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), in arm_64_lpae_alloc_pgtable_s2()
947 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s2()
954 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s2()
955 return &data->iop; in arm_64_lpae_alloc_pgtable_s2()
958 kfree(data); in arm_64_lpae_alloc_pgtable_s2()
965 if (cfg->ias > 32 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s1()
968 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s1()
975 if (cfg->ias > 40 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s2()
978 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s2()
985 struct arm_lpae_io_pgtable *data; in arm_mali_lpae_alloc_pgtable() local
988 if (cfg->quirks) in arm_mali_lpae_alloc_pgtable()
991 if (cfg->ias > 48 || cfg->oas > 40) in arm_mali_lpae_alloc_pgtable()
994 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_mali_lpae_alloc_pgtable()
996 data = arm_lpae_alloc_pgtable(cfg); in arm_mali_lpae_alloc_pgtable()
997 if (!data) in arm_mali_lpae_alloc_pgtable()
1000 /* Mali seems to need a full 4-level table regardless of IAS */ in arm_mali_lpae_alloc_pgtable()
1001 if (data->start_level > 0) { in arm_mali_lpae_alloc_pgtable()
1002 data->start_level = 0; in arm_mali_lpae_alloc_pgtable()
1003 data->pgd_bits = 0; in arm_mali_lpae_alloc_pgtable()
1006 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the in arm_mali_lpae_alloc_pgtable()
1007 * best we can do is mimic the out-of-tree driver and hope that the in arm_mali_lpae_alloc_pgtable()
1008 * "implementation-defined caching policy" is good enough. Similarly, in arm_mali_lpae_alloc_pgtable()
1012 cfg->arm_mali_lpae_cfg.memattr = in arm_mali_lpae_alloc_pgtable()
1020 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, in arm_mali_lpae_alloc_pgtable()
1022 if (!data->pgd) in arm_mali_lpae_alloc_pgtable()
1028 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | in arm_mali_lpae_alloc_pgtable()
1031 if (cfg->coherent_walk) in arm_mali_lpae_alloc_pgtable()
1032 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; in arm_mali_lpae_alloc_pgtable()
1034 return &data->iop; in arm_mali_lpae_alloc_pgtable()
1037 kfree(data); in arm_mali_lpae_alloc_pgtable()
1075 static void __init dummy_tlb_flush(unsigned long iova, size_t size, in dummy_tlb_flush() argument
1079 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); in dummy_tlb_flush()
1098 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_dump_ops() local
1099 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_dump_ops()
1101 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", in arm_lpae_dump_ops()
1102 cfg->pgsize_bitmap, cfg->ias); in arm_lpae_dump_ops()
1103 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n", in arm_lpae_dump_ops()
1104 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data), in arm_lpae_dump_ops()
1105 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd); in arm_lpae_dump_ops()
1112 -EFAULT; \
1124 size_t size; in arm_lpae_run_tests() local
1134 return -ENOMEM; in arm_lpae_run_tests()
1141 if (ops->iova_to_phys(ops, 42)) in arm_lpae_run_tests()
1144 if (ops->iova_to_phys(ops, SZ_1G + 42)) in arm_lpae_run_tests()
1147 if (ops->iova_to_phys(ops, SZ_2G + 42)) in arm_lpae_run_tests()
1154 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1155 size = 1UL << j; in arm_lpae_run_tests()
1157 if (ops->map(ops, iova, iova, size, IOMMU_READ | in arm_lpae_run_tests()
1164 if (!ops->map(ops, iova, iova + size, size, in arm_lpae_run_tests()
1168 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_lpae_run_tests()
1175 size = 1UL << __ffs(cfg->pgsize_bitmap); in arm_lpae_run_tests()
1176 if (ops->unmap(ops, SZ_1G + size, size, NULL) != size) in arm_lpae_run_tests()
1180 if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL)) in arm_lpae_run_tests()
1183 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) in arm_lpae_run_tests()
1188 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1189 size = 1UL << j; in arm_lpae_run_tests()
1191 if (ops->unmap(ops, iova, size, NULL) != size) in arm_lpae_run_tests()
1194 if (ops->iova_to_phys(ops, iova + 42)) in arm_lpae_run_tests()
1198 if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL)) in arm_lpae_run_tests()
1201 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_lpae_run_tests()
1247 return fail ? -EFAULT : 0; in arm_lpae_do_selftests()