1348 	u64 *pte;  in increase_address_space()  local
1356 	pte = (void *)get_zeroed_page(gfp);  in increase_address_space()
1357 	if (!pte)  in increase_address_space()
1360 	*pte             = PM_LEVEL_PDE(domain->mode,  in increase_address_space()
1362 	domain->pt_root  = pte;  in increase_address_space()
1379 	u64 *pte, *page;  in alloc_pte()  local
1387 	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1394 		__pte = *pte;  in alloc_pte()
1404 			if (cmpxchg64(pte, __pte, __npte) != __pte) {  in alloc_pte()
1411 		if (PM_PTE_LEVEL(*pte) != level)  in alloc_pte()
1416 		pte = IOMMU_PTE_PAGE(*pte);  in alloc_pte()
1419 			*pte_page = pte;  in alloc_pte()
1421 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1424 	return pte;  in alloc_pte()
1436 	u64 *pte;  in fetch_pte()  local
1442 	pte	   = &domain->pt_root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1448 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
1452 		if (PM_PTE_LEVEL(*pte) == 7 ||  in fetch_pte()
1453 		    PM_PTE_LEVEL(*pte) == 0)  in fetch_pte()
1457 		if (PM_PTE_LEVEL(*pte) != level)  in fetch_pte()
1463 		pte	   = IOMMU_PTE_PAGE(*pte);  in fetch_pte()
1464 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1468 	if (PM_PTE_LEVEL(*pte) == 0x07) {  in fetch_pte()
1475 		*page_size = pte_mask = PTE_PAGE_SIZE(*pte);  in fetch_pte()
1477 		pte        = (u64 *)(((unsigned long)pte) & pte_mask);  in fetch_pte()
1480 	return pte;  in fetch_pte()
1497 	u64 __pte, *pte;  in iommu_map_page()  local
1507 	pte   = alloc_pte(dom, bus_addr, page_size, NULL, gfp);  in iommu_map_page()
1509 	if (!pte)  in iommu_map_page()
1513 		if (IOMMU_PTE_PRESENT(pte[i]))  in iommu_map_page()
1528 		pte[i] = __pte;  in iommu_map_page()
1541 	u64 *pte;  in iommu_unmap_page()  local
1549 		pte = fetch_pte(dom, bus_addr, &unmap_size);  in iommu_unmap_page()
1551 		if (pte) {  in iommu_unmap_page()
1556 				pte[i] = 0ULL;  in iommu_unmap_page()
3126 	u64 *pte, __pte;  in amd_iommu_iova_to_phys()  local
3131 	pte = fetch_pte(domain, iova, &pte_pgsize);  in amd_iommu_iova_to_phys()
3133 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in amd_iommu_iova_to_phys()
3137 	__pte	    = __sme_clr(*pte & PM_ADDR_MASK);  in amd_iommu_iova_to_phys()
3454 	u64 *pte;  in __get_gcr3_pte()  local
3459 		pte   = &root[index];  in __get_gcr3_pte()
3464 		if (!(*pte & GCR3_VALID)) {  in __get_gcr3_pte()
3472 			*pte = iommu_virt_to_phys(root) | GCR3_VALID;  in __get_gcr3_pte()
3475 		root = iommu_phys_to_virt(*pte & PAGE_MASK);  in __get_gcr3_pte()
3480 	return pte;  in __get_gcr3_pte()
3486 	u64 *pte;  in __set_gcr3()  local
3491 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);  in __set_gcr3()
3492 	if (pte == NULL)  in __set_gcr3()
3495 	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;  in __set_gcr3()
3502 	u64 *pte;  in __clear_gcr3()  local
3507 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);  in __clear_gcr3()
3508 	if (pte == NULL)  in __clear_gcr3()
3511 	*pte = 0;  in __clear_gcr3()