29 	container_of((x), struct arm_lpae_io_pgtable, iop)
181 	struct io_pgtable	iop;  member
305 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in __arm_lpae_init_pte()
308 	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_init_pte()
313 	if (data->iop.fmt != ARM_MALI_LPAE)  in __arm_lpae_init_pte()
318 	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);  in __arm_lpae_init_pte()
328 	if (iopte_leaf(pte, lvl, data->iop.fmt)) {  in arm_lpae_init_pte()
389 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()
416 	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {  in __arm_lpae_map()
433 	if (data->iop.fmt == ARM_64_LPAE_S1 ||  in arm_lpae_prot_to_pte()
434 	    data->iop.fmt == ARM_32_LPAE_S1) {  in arm_lpae_prot_to_pte()
452 	if (data->iop.fmt == ARM_64_LPAE_S2 ||  in arm_lpae_prot_to_pte()
453 	    data->iop.fmt == ARM_32_LPAE_S2) {  in arm_lpae_prot_to_pte()
490 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||  in arm_lpae_map()
491 		    paddr >= (1ULL << data->iop.cfg.oas)))  in arm_lpae_map()
527 		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))  in __arm_lpae_free_pgtable()
533 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg);  in __arm_lpae_free_pgtable()
536 static void arm_lpae_free_pgtable(struct io_pgtable *iop)  in arm_lpae_free_pgtable()  argument
538 	struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);  in arm_lpae_free_pgtable()
550 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_split_blk_unmap()
591 		io_pgtable_tlb_add_page(&data->iop, gather, iova, size);  in arm_lpae_split_blk_unmap()
604 	struct io_pgtable *iop = &data->iop;  in __arm_lpae_unmap()  local
617 		__arm_lpae_set_pte(ptep, 0, &iop->cfg);  in __arm_lpae_unmap()
619 		if (!iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
621 			io_pgtable_tlb_flush_walk(iop, iova, size,  in __arm_lpae_unmap()
625 		} else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {  in __arm_lpae_unmap()
633 			io_pgtable_tlb_add_page(iop, gather, iova, size);  in __arm_lpae_unmap()
637 	} else if (iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
658 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))  in arm_lpae_unmap()
685 		if (iopte_leaf(pte, lvl, data->iop.fmt))  in arm_lpae_iova_to_phys()
779 	data->iop.ops = (struct io_pgtable_ops) {  in arm_lpae_alloc_pgtable()
881 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s1()
977 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s2()
987 	struct io_pgtable *iop;  in arm_32_lpae_alloc_pgtable_s1()  local
993 	iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
994 	if (iop) {  in arm_32_lpae_alloc_pgtable_s1()
999 	return iop;  in arm_32_lpae_alloc_pgtable_s1()
1005 	struct io_pgtable *iop;  in arm_32_lpae_alloc_pgtable_s2()  local
1011 	iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
1012 	if (iop)  in arm_32_lpae_alloc_pgtable_s2()
1015 	return iop;  in arm_32_lpae_alloc_pgtable_s2()
1066 	return &data->iop;  in arm_mali_lpae_alloc_pgtable()
1130 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()