Lines Matching refs:pte
1257 u64 *pte; in increase_address_space() local
1263 pte = (void *)get_zeroed_page(gfp); in increase_address_space()
1264 if (!pte) in increase_address_space()
1267 *pte = PM_LEVEL_PDE(domain->mode, in increase_address_space()
1269 domain->pt_root = pte; in increase_address_space()
1283 u64 *pte, *page; in alloc_pte() local
1291 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; in alloc_pte()
1296 if (!IOMMU_PTE_PRESENT(*pte)) { in alloc_pte()
1300 *pte = PM_LEVEL_PDE(level, virt_to_phys(page)); in alloc_pte()
1304 if (PM_PTE_LEVEL(*pte) != level) in alloc_pte()
1309 pte = IOMMU_PTE_PAGE(*pte); in alloc_pte()
1312 *pte_page = pte; in alloc_pte()
1314 pte = &pte[PM_LEVEL_INDEX(level, address)]; in alloc_pte()
1317 return pte; in alloc_pte()
1327 u64 *pte; in fetch_pte() local
1333 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
1338 if (!IOMMU_PTE_PRESENT(*pte)) in fetch_pte()
1342 if (PM_PTE_LEVEL(*pte) == 0x07) { in fetch_pte()
1349 pte_mask = PTE_PAGE_SIZE(*pte); in fetch_pte()
1351 __pte = ((unsigned long)pte) & pte_mask; in fetch_pte()
1357 if (PM_PTE_LEVEL(*pte) != level) in fetch_pte()
1363 pte = IOMMU_PTE_PAGE(*pte); in fetch_pte()
1364 pte = &pte[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
1367 return pte; in fetch_pte()
1383 u64 __pte, *pte; in iommu_map_page() local
1392 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); in iommu_map_page()
1394 if (!pte) in iommu_map_page()
1398 if (IOMMU_PTE_PRESENT(pte[i])) in iommu_map_page()
1413 pte[i] = __pte; in iommu_map_page()
1425 u64 *pte; in iommu_unmap_page() local
1433 pte = fetch_pte(dom, bus_addr); in iommu_unmap_page()
1435 if (!pte) { in iommu_unmap_page()
1441 } else if (PM_PTE_LEVEL(*pte) == 0) { in iommu_unmap_page()
1444 *pte = 0ULL; in iommu_unmap_page()
1449 unmap_size = PTE_PAGE_SIZE(*pte); in iommu_unmap_page()
1456 pte[i] = 0ULL; in iommu_unmap_page()
1624 u64 *pte, *pte_page; in alloc_new_range() local
1627 pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE, in alloc_new_range()
1629 if (!pte) in alloc_new_range()
1676 u64 *pte = fetch_pte(&dma_dom->domain, i); in alloc_new_range() local
1677 if (!pte || !IOMMU_PTE_PRESENT(*pte)) in alloc_new_range()
2569 u64 *pte, *pte_page; in dma_ops_get_pte() local
2575 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; in dma_ops_get_pte()
2576 if (!pte) { in dma_ops_get_pte()
2577 pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page, in dma_ops_get_pte()
2581 pte += PM_LEVEL_INDEX(0, address); in dma_ops_get_pte()
2585 return pte; in dma_ops_get_pte()
2597 u64 *pte, __pte; in dma_ops_domain_map() local
2603 pte = dma_ops_get_pte(dom, address); in dma_ops_domain_map()
2604 if (!pte) in dma_ops_domain_map()
2616 WARN_ON(*pte); in dma_ops_domain_map()
2618 *pte = __pte; in dma_ops_domain_map()
2630 u64 *pte; in dma_ops_domain_unmap() local
2639 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; in dma_ops_domain_unmap()
2640 if (!pte) in dma_ops_domain_unmap()
2643 pte += PM_LEVEL_INDEX(0, address); in dma_ops_domain_unmap()
2645 WARN_ON(!*pte); in dma_ops_domain_unmap()
2647 *pte = 0ULL; in dma_ops_domain_unmap()
3395 u64 *pte, __pte; in amd_iommu_iova_to_phys() local
3400 pte = fetch_pte(domain, iova); in amd_iommu_iova_to_phys()
3402 if (!pte || !IOMMU_PTE_PRESENT(*pte)) in amd_iommu_iova_to_phys()
3405 if (PM_PTE_LEVEL(*pte) == 0) in amd_iommu_iova_to_phys()
3408 offset_mask = PTE_PAGE_SIZE(*pte) - 1; in amd_iommu_iova_to_phys()
3410 __pte = *pte & PM_ADDR_MASK; in amd_iommu_iova_to_phys()
3662 u64 *pte; in __get_gcr3_pte() local
3667 pte = &root[index]; in __get_gcr3_pte()
3672 if (!(*pte & GCR3_VALID)) { in __get_gcr3_pte()
3680 *pte = __pa(root) | GCR3_VALID; in __get_gcr3_pte()
3683 root = __va(*pte & PAGE_MASK); in __get_gcr3_pte()
3688 return pte; in __get_gcr3_pte()
3694 u64 *pte; in __set_gcr3() local
3699 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true); in __set_gcr3()
3700 if (pte == NULL) in __set_gcr3()
3703 *pte = (cr3 & PAGE_MASK) | GCR3_VALID; in __set_gcr3()
3710 u64 *pte; in __clear_gcr3() local
3715 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false); in __clear_gcr3()
3716 if (pte == NULL) in __clear_gcr3()
3719 *pte = 0; in __clear_gcr3()