198 				    struct io_pgtable_cfg *cfg)  in __arm_lpae_alloc_pages()  argument
200 	struct device *dev = cfg->iommu_dev;  in __arm_lpae_alloc_pages()
213 	if (!cfg->coherent_walk) {  in __arm_lpae_alloc_pages()
237 				  struct io_pgtable_cfg *cfg)  in __arm_lpae_free_pages()  argument
239 	if (!cfg->coherent_walk)  in __arm_lpae_free_pages()
240 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),  in __arm_lpae_free_pages()
246 				struct io_pgtable_cfg *cfg)  in __arm_lpae_sync_pte()  argument
248 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
252 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)  in __arm_lpae_clear_pte()  argument
257 	if (!cfg->coherent_walk)  in __arm_lpae_clear_pte()
258 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_clear_pte()
271 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_init_pte()  local
283 	if (!cfg->coherent_walk)  in __arm_lpae_init_pte()
284 		__arm_lpae_sync_pte(ptep, num_entries, cfg);  in __arm_lpae_init_pte()
325 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_install_table()  local
328 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_install_table()
340 	if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))  in arm_lpae_install_table()
344 	__arm_lpae_sync_pte(ptep, 1, cfg);  in arm_lpae_install_table()
359 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()  local
384 		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg);  in __arm_lpae_map()
390 			__arm_lpae_free_pages(cptep, tblsz, cfg);  in __arm_lpae_map()
391 	} else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {  in __arm_lpae_map()
392 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_map()
482 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_prot_to_pte()
496 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_map_pages()  local
500 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_map_pages()
502 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))  in arm_lpae_map_pages()
505 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_map_pages()
507 	if (WARN_ON(iaext || paddr >> cfg->oas))  in arm_lpae_map_pages()
561 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg);  in __arm_lpae_free_pgtable()
578 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_split_blk_unmap()  local
589 	tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg);  in arm_lpae_split_blk_unmap()
612 		__arm_lpae_free_pages(tablep, tablesz, cfg);  in arm_lpae_split_blk_unmap()
661 			__arm_lpae_clear_pte(ptep, &iop->cfg);  in __arm_lpae_unmap()
696 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_unmap_pages()  local
698 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_unmap_pages()
700 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))  in arm_lpae_unmap_pages()
703 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_unmap_pages()
754 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)  in arm_lpae_restrict_pgsizes()  argument
766 	if (cfg->pgsize_bitmap & PAGE_SIZE)  in arm_lpae_restrict_pgsizes()
768 	else if (cfg->pgsize_bitmap & ~PAGE_MASK)  in arm_lpae_restrict_pgsizes()
769 		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);  in arm_lpae_restrict_pgsizes()
770 	else if (cfg->pgsize_bitmap & PAGE_MASK)  in arm_lpae_restrict_pgsizes()
771 		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);  in arm_lpae_restrict_pgsizes()
785 		if (cfg->oas > 48)  in arm_lpae_restrict_pgsizes()
792 	cfg->pgsize_bitmap &= page_sizes;  in arm_lpae_restrict_pgsizes()
793 	cfg->ias = min(cfg->ias, max_addr_bits);  in arm_lpae_restrict_pgsizes()
794 	cfg->oas = min(cfg->oas, max_addr_bits);  in arm_lpae_restrict_pgsizes()
798 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)  in arm_lpae_alloc_pgtable()  argument
803 	arm_lpae_restrict_pgsizes(cfg);  in arm_lpae_alloc_pgtable()
805 	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))  in arm_lpae_alloc_pgtable()
808 	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
811 	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
818 	pg_shift = __ffs(cfg->pgsize_bitmap);  in arm_lpae_alloc_pgtable()
821 	va_bits = cfg->ias - pg_shift;  in arm_lpae_alloc_pgtable()
840 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s1()  argument
844 	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;  in arm_64_lpae_alloc_pgtable_s1()
847 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_64_lpae_alloc_pgtable_s1()
852 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s1()
857 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s1()
861 		if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)  in arm_64_lpae_alloc_pgtable_s1()
866 		if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))  in arm_64_lpae_alloc_pgtable_s1()
872 	tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;  in arm_64_lpae_alloc_pgtable_s1()
885 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s1()
911 	tcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s1()
927 	cfg->arm_lpae_s1_cfg.mair = reg;  in arm_64_lpae_alloc_pgtable_s1()
931 					   GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s1()
939 	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s1()
948 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s2()  argument
952 	typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;  in arm_64_lpae_alloc_pgtable_s2()
955 	if (cfg->quirks)  in arm_64_lpae_alloc_pgtable_s2()
958 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s2()
977 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s2()
1002 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s2()
1028 	vtcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s2()
1033 					   GFP_KERNEL, cfg);  in arm_64_lpae_alloc_pgtable_s2()
1041 	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s2()
1050 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s1()  argument
1052 	if (cfg->ias > 32 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s1()
1055 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s1()
1056 	return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
1060 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s2()  argument
1062 	if (cfg->ias > 40 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s2()
1065 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s2()
1066 	return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
1070 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)  in arm_mali_lpae_alloc_pgtable()  argument
1075 	if (cfg->quirks)  in arm_mali_lpae_alloc_pgtable()
1078 	if (cfg->ias > 48 || cfg->oas > 40)  in arm_mali_lpae_alloc_pgtable()
1081 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_mali_lpae_alloc_pgtable()
1083 	data = arm_lpae_alloc_pgtable(cfg);  in arm_mali_lpae_alloc_pgtable()
1099 	cfg->arm_mali_lpae_cfg.memattr =  in arm_mali_lpae_alloc_pgtable()
1108 					   cfg);  in arm_mali_lpae_alloc_pgtable()
1115 	cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |  in arm_mali_lpae_alloc_pgtable()
1118 	if (cfg->coherent_walk)  in arm_mali_lpae_alloc_pgtable()
1119 		cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;  in arm_mali_lpae_alloc_pgtable()
1129 apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)  in apple_dart_alloc_pgtable()  argument
1134 	if (cfg->oas > 36)  in apple_dart_alloc_pgtable()
1137 	data = arm_lpae_alloc_pgtable(cfg);  in apple_dart_alloc_pgtable()
1155 	cfg->apple_dart_cfg.n_ttbrs = 1 << data->pgd_bits;  in apple_dart_alloc_pgtable()
1159 					   cfg);  in apple_dart_alloc_pgtable()
1163 	for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i)  in apple_dart_alloc_pgtable()
1164 		cfg->apple_dart_cfg.ttbr[i] =  in apple_dart_alloc_pgtable()
1236 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()  local
1239 		cfg->pgsize_bitmap, cfg->ias);  in arm_lpae_dump_ops()
1252 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)  in arm_lpae_run_tests()  argument
1267 		cfg_cookie = cfg;  in arm_lpae_run_tests()
1268 		ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);  in arm_lpae_run_tests()
1291 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1312 		size = 1UL << __ffs(cfg->pgsize_bitmap);  in arm_lpae_run_tests()
1325 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1364 	struct io_pgtable_cfg cfg = {  in arm_lpae_do_selftests()  local
1372 			cfg.pgsize_bitmap = pgsize[i];  in arm_lpae_do_selftests()
1373 			cfg.ias = ias[j];  in arm_lpae_do_selftests()
1376 			if (arm_lpae_run_tests(&cfg))  in arm_lpae_do_selftests()