51 #define _ARM_V7S_LVL_BITS(lvl, cfg)	((lvl) == 1 ? ((cfg)->ias - 20) : 8)  argument
55 #define ARM_V7S_PTES_PER_LVL(lvl, cfg)	(1 << _ARM_V7S_LVL_BITS(lvl, cfg))  argument
56 #define ARM_V7S_TABLE_SIZE(lvl, cfg)						\  argument
57 	(ARM_V7S_PTES_PER_LVL(lvl, cfg) * sizeof(arm_v7s_iopte))
62 #define _ARM_V7S_IDX_MASK(lvl, cfg)	(ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1)  argument
63 #define ARM_V7S_LVL_IDX(addr, lvl, cfg)	({				\  argument
65 	((addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l, cfg); \
179 static bool arm_v7s_is_mtk_enabled(struct io_pgtable_cfg *cfg)  in arm_v7s_is_mtk_enabled()  argument
182 		(cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT);  in arm_v7s_is_mtk_enabled()
197 				    struct io_pgtable_cfg *cfg)  in paddr_to_iopte()  argument
201 	if (arm_v7s_is_mtk_enabled(cfg))  in paddr_to_iopte()
208 				  struct io_pgtable_cfg *cfg)  in iopte_to_paddr()  argument
221 	if (!arm_v7s_is_mtk_enabled(cfg))  in iopte_to_paddr()
236 	return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));  in iopte_deref()
242 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_alloc_table()  local
243 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_alloc_table()
246 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_alloc_table()
254 	gfp_l1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in __arm_v7s_alloc_table()
266 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in __arm_v7s_alloc_table()
267 	    phys >= (1ULL << cfg->oas) : phys != (arm_v7s_iopte)phys) {  in __arm_v7s_alloc_table()
272 	if (!cfg->coherent_walk) {  in __arm_v7s_alloc_table()
302 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_free_table()  local
303 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_free_table()
304 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_free_table()
306 	if (!cfg->coherent_walk)  in __arm_v7s_free_table()
316 			       struct io_pgtable_cfg *cfg)  in __arm_v7s_pte_sync()  argument
318 	if (cfg->coherent_walk)  in __arm_v7s_pte_sync()
321 	dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep),  in __arm_v7s_pte_sync()
325 			      int num_entries, struct io_pgtable_cfg *cfg)  in __arm_v7s_set_pte()  argument
332 	__arm_v7s_pte_sync(ptep, num_entries, cfg);  in __arm_v7s_set_pte()
336 					 struct io_pgtable_cfg *cfg)  in arm_v7s_prot_to_pte()  argument
338 	bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS);  in arm_v7s_prot_to_pte()
360 	if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))  in arm_v7s_prot_to_pte()
435 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_init_pte()  local
448 			tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg);  in arm_v7s_init_pte()
458 	pte = arm_v7s_prot_to_pte(prot, lvl, cfg);  in arm_v7s_init_pte()
462 	pte |= paddr_to_iopte(paddr, lvl, cfg);  in arm_v7s_init_pte()
464 	__arm_v7s_set_pte(ptep, pte, num_entries, cfg);  in arm_v7s_init_pte()
471 					   struct io_pgtable_cfg *cfg)  in arm_v7s_install_table()  argument
478 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT)  in arm_v7s_install_table()
481 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_v7s_install_table()
492 	__arm_v7s_pte_sync(ptep, 1, cfg);  in arm_v7s_install_table()
501 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_map()  local
506 	ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg);  in __arm_v7s_map()
524 		pte = arm_v7s_install_table(cptep, ptep, 0, cfg);  in __arm_v7s_map()
529 		__arm_v7s_pte_sync(ptep, 1, cfg);  in __arm_v7s_map()
551 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||  in arm_v7s_map_pages()
552 		    paddr >= (1ULL << data->iop.cfg.oas)))  in arm_v7s_map_pages()
590 	for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) {  in arm_v7s_free_pgtable()
621 	__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);  in arm_v7s_split_cont()
634 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_split_blk_unmap()  local
642 	num_ptes = ARM_V7S_PTES_PER_LVL(2, cfg);  in arm_v7s_split_blk_unmap()
644 	unmap_idx = ARM_V7S_LVL_IDX(iova, 2, cfg);  in arm_v7s_split_blk_unmap()
646 	pte = arm_v7s_prot_to_pte(arm_v7s_pte_to_prot(blk_pte, 1), 2, cfg);  in arm_v7s_split_blk_unmap()
655 		__arm_v7s_set_pte(&tablep[i], pte, num_entries, cfg);  in arm_v7s_split_blk_unmap()
658 	pte = arm_v7s_install_table(tablep, ptep, blk_pte, cfg);  in arm_v7s_split_blk_unmap()
686 	idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg);  in __arm_v7s_unmap()
716 		__arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg);  in __arm_v7s_unmap()
752 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))  in arm_v7s_unmap_pages()
782 		ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg);  in arm_v7s_iova_to_phys()
793 	return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);  in arm_v7s_iova_to_phys()
796 static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,  in arm_v7s_alloc_pgtable()  argument
803 	if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
806 	if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
809 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_v7s_alloc_pgtable()
816 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT &&  in arm_v7s_alloc_pgtable()
817 	    !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS))  in arm_v7s_alloc_pgtable()
820 	if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) &&  in arm_v7s_alloc_pgtable()
821 	    !arm_v7s_is_mtk_enabled(cfg))  in arm_v7s_alloc_pgtable()
834 	slab_flag = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in arm_v7s_alloc_pgtable()
838 					    ARM_V7S_TABLE_SIZE(2, cfg),  in arm_v7s_alloc_pgtable()
839 					    ARM_V7S_TABLE_SIZE(2, cfg),  in arm_v7s_alloc_pgtable()
853 	data->iop.cfg = *cfg;  in arm_v7s_alloc_pgtable()
859 	cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M;  in arm_v7s_alloc_pgtable()
862 	cfg->arm_v7s_cfg.tcr = 0;  in arm_v7s_alloc_pgtable()
869 	cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) |  in arm_v7s_alloc_pgtable()
874 	cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) |  in arm_v7s_alloc_pgtable()
887 	if (arm_v7s_is_mtk_enabled(cfg))  in arm_v7s_alloc_pgtable()
888 		cfg->arm_v7s_cfg.ttbr = paddr | upper_32_bits(paddr);  in arm_v7s_alloc_pgtable()
890 		cfg->arm_v7s_cfg.ttbr = paddr | ARM_V7S_TTBR_S |  in arm_v7s_alloc_pgtable()
891 					(cfg->coherent_walk ? (ARM_V7S_TTBR_NOS |  in arm_v7s_alloc_pgtable()
947 	struct io_pgtable_cfg cfg = {  in arm_v7s_do_selftests()  local
960 	cfg_cookie = &cfg;  in arm_v7s_do_selftests()
962 	ops = alloc_io_pgtable_ops(ARM_V7S, &cfg, &cfg);  in arm_v7s_do_selftests()
985 	for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {  in arm_v7s_do_selftests()
1007 	size = 1UL << __ffs(cfg.pgsize_bitmap);  in arm_v7s_do_selftests()
1025 	for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {  in arm_v7s_do_selftests()