170 #define iopte_deref(pte,d)					\  argument
171 	(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)	\
174 #define iopte_type(pte,l)					\  argument
175 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
177 #define iopte_prot(pte)	((pte) & ARM_LPAE_PTE_ATTR_MASK)  argument
179 #define iopte_leaf(pte,l)					\  argument
181 		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) :	\
182 		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK))
184 #define iopte_to_pfn(pte,d)					\  argument
185 	(((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift)
252 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,  in __arm_lpae_set_pte()  argument
255 	*ptep = pte;  in __arm_lpae_set_pte()
260 					   sizeof(pte), DMA_TO_DEVICE);  in __arm_lpae_set_pte()
272 	arm_lpae_iopte pte = prot;  in arm_lpae_init_pte()  local
293 		pte |= ARM_LPAE_PTE_NS;  in arm_lpae_init_pte()
296 		pte |= ARM_LPAE_PTE_TYPE_PAGE;  in arm_lpae_init_pte()
298 		pte |= ARM_LPAE_PTE_TYPE_BLOCK;  in arm_lpae_init_pte()
300 	pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;  in arm_lpae_init_pte()
301 	pte |= pfn_to_iopte(paddr >> data->pg_shift, data);  in arm_lpae_init_pte()
303 	__arm_lpae_set_pte(ptep, pte, cfg);  in arm_lpae_init_pte()
311 	arm_lpae_iopte *cptep, pte;  in __arm_lpae_map()  local
327 	pte = *ptep;  in __arm_lpae_map()
328 	if (!pte) {  in __arm_lpae_map()
334 		pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE;  in __arm_lpae_map()
336 			pte |= ARM_LPAE_PTE_NSTABLE;  in __arm_lpae_map()
337 		__arm_lpae_set_pte(ptep, pte, cfg);  in __arm_lpae_map()
338 	} else if (!iopte_leaf(pte, lvl)) {  in __arm_lpae_map()
339 		cptep = iopte_deref(pte, data);  in __arm_lpae_map()
353 	arm_lpae_iopte pte;  in arm_lpae_prot_to_pte()  local
357 		pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG;  in arm_lpae_prot_to_pte()
360 			pte |= ARM_LPAE_PTE_AP_RDONLY;  in arm_lpae_prot_to_pte()
363 			pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE  in arm_lpae_prot_to_pte()
366 		pte = ARM_LPAE_PTE_HAP_FAULT;  in arm_lpae_prot_to_pte()
368 			pte |= ARM_LPAE_PTE_HAP_READ;  in arm_lpae_prot_to_pte()
370 			pte |= ARM_LPAE_PTE_HAP_WRITE;  in arm_lpae_prot_to_pte()
372 			pte |= ARM_LPAE_PTE_MEMATTR_OIWB;  in arm_lpae_prot_to_pte()
374 			pte |= ARM_LPAE_PTE_MEMATTR_NC;  in arm_lpae_prot_to_pte()
378 		pte |= ARM_LPAE_PTE_XN;  in arm_lpae_prot_to_pte()
380 	return pte;  in arm_lpae_prot_to_pte()
426 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()  local
428 		if (!pte || iopte_leaf(pte, lvl))  in __arm_lpae_free_pgtable()
431 		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));  in __arm_lpae_free_pgtable()
489 	arm_lpae_iopte pte;  in __arm_lpae_unmap()  local
495 	pte = *ptep;  in __arm_lpae_unmap()
498 	if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS)))  in __arm_lpae_unmap()
505 		if (!iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
509 			ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
516 	} else if (iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
522 						iopte_prot(pte), lvl, ptep,  in __arm_lpae_unmap()
527 	ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
551 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()  local
560 		pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data));  in arm_lpae_iova_to_phys()
563 		if (!pte)  in arm_lpae_iova_to_phys()
567 		if (iopte_leaf(pte,lvl))  in arm_lpae_iova_to_phys()
571 		ptep = iopte_deref(pte, data);  in arm_lpae_iova_to_phys()
579 	return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;  in arm_lpae_iova_to_phys()