Lines Matching refs:pte
331 static inline void dma_clear_pte(struct dma_pte *pte) in dma_clear_pte() argument
333 pte->val = 0; in dma_clear_pte()
336 static inline u64 dma_pte_addr(struct dma_pte *pte) in dma_pte_addr() argument
339 return pte->val & VTD_PAGE_MASK; in dma_pte_addr()
342 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK; in dma_pte_addr()
346 static inline bool dma_pte_present(struct dma_pte *pte) in dma_pte_present() argument
348 return (pte->val & 3) != 0; in dma_pte_present()
351 static inline bool dma_pte_superpage(struct dma_pte *pte) in dma_pte_superpage() argument
353 return (pte->val & DMA_PTE_LARGE_PAGE); in dma_pte_superpage()
356 static inline int first_pte_in_page(struct dma_pte *pte) in first_pte_in_page() argument
358 return !((unsigned long)pte & ~VTD_PAGE_MASK); in first_pte_in_page()
991 struct dma_pte *parent, *pte = NULL; in pfn_to_dma_pte() local
1007 pte = &parent[offset]; in pfn_to_dma_pte()
1008 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte))) in pfn_to_dma_pte()
1013 if (!dma_pte_present(pte)) { in pfn_to_dma_pte()
1023 if (cmpxchg64(&pte->val, 0ULL, pteval)) in pfn_to_dma_pte()
1027 domain_flush_cache(domain, pte, sizeof(*pte)); in pfn_to_dma_pte()
1032 parent = phys_to_virt(dma_pte_addr(pte)); in pfn_to_dma_pte()
1039 return pte; in pfn_to_dma_pte()
1048 struct dma_pte *parent, *pte = NULL; in dma_pfn_level_pte() local
1055 pte = &parent[offset]; in dma_pfn_level_pte()
1057 return pte; in dma_pfn_level_pte()
1059 if (!dma_pte_present(pte)) { in dma_pfn_level_pte()
1064 if (dma_pte_superpage(pte)) { in dma_pfn_level_pte()
1066 return pte; in dma_pfn_level_pte()
1069 parent = phys_to_virt(dma_pte_addr(pte)); in dma_pfn_level_pte()
1081 struct dma_pte *first_pte, *pte; in dma_pte_clear_range() local
1090 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page); in dma_pte_clear_range()
1091 if (!pte) { in dma_pte_clear_range()
1096 dma_clear_pte(pte); in dma_pte_clear_range()
1098 pte++; in dma_pte_clear_range()
1099 } while (start_pfn <= last_pfn && !first_pte_in_page(pte)); in dma_pte_clear_range()
1102 (void *)pte - (void *)first_pte); in dma_pte_clear_range()
1108 int retain_level, struct dma_pte *pte, in dma_pte_free_level() argument
1113 pte = &pte[pfn_level_offset(pfn, level)]; in dma_pte_free_level()
1119 if (!dma_pte_present(pte) || dma_pte_superpage(pte)) in dma_pte_free_level()
1123 level_pte = phys_to_virt(dma_pte_addr(pte)); in dma_pte_free_level()
1137 dma_clear_pte(pte); in dma_pte_free_level()
1138 domain_flush_cache(domain, pte, sizeof(*pte)); in dma_pte_free_level()
1143 } while (!first_pte_in_page(++pte) && pfn <= last_pfn); in dma_pte_free_level()
1179 int level, struct dma_pte *pte, in dma_pte_list_pagetables() argument
1184 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT); in dma_pte_list_pagetables()
1191 pte = page_address(pg); in dma_pte_list_pagetables()
1193 if (dma_pte_present(pte) && !dma_pte_superpage(pte)) in dma_pte_list_pagetables()
1195 pte, freelist); in dma_pte_list_pagetables()
1196 pte++; in dma_pte_list_pagetables()
1197 } while (!first_pte_in_page(pte)); in dma_pte_list_pagetables()
1203 struct dma_pte *pte, unsigned long pfn, in dma_pte_clear_level() argument
1211 pte = &pte[pfn_level_offset(pfn, level)]; in dma_pte_clear_level()
1216 if (!dma_pte_present(pte)) in dma_pte_clear_level()
1226 if (level > 1 && !dma_pte_superpage(pte)) in dma_pte_clear_level()
1227 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist); in dma_pte_clear_level()
1229 dma_clear_pte(pte); in dma_pte_clear_level()
1231 first_pte = pte; in dma_pte_clear_level()
1232 last_pte = pte; in dma_pte_clear_level()
1236 phys_to_virt(dma_pte_addr(pte)), in dma_pte_clear_level()
1242 } while (!first_pte_in_page(++pte) && pfn <= last_pfn); in dma_pte_clear_level()
2252 struct dma_pte *first_pte = NULL, *pte = NULL; in __domain_mapping() local
2283 if (!pte) { in __domain_mapping()
2286 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl); in __domain_mapping()
2287 if (!pte) in __domain_mapping()
2315 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); in __domain_mapping()
2349 pte++; in __domain_mapping()
2350 if (!nr_pages || first_pte_in_page(pte) || in __domain_mapping()
2353 (void *)pte - (void *)first_pte); in __domain_mapping()
2354 pte = NULL; in __domain_mapping()
5021 struct dma_pte *pte; in intel_iommu_attach_device() local
5023 pte = dmar_domain->pgd; in intel_iommu_attach_device()
5024 if (dma_pte_present(pte)) { in intel_iommu_attach_device()
5026 phys_to_virt(dma_pte_addr(pte)); in intel_iommu_attach_device()
5027 free_pgtable_page(pte); in intel_iommu_attach_device()
5122 struct dma_pte *pte; in intel_iommu_iova_to_phys() local
5126 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level); in intel_iommu_iova_to_phys()
5127 if (pte && dma_pte_present(pte)) in intel_iommu_iova_to_phys()
5128 phys = dma_pte_addr(pte) + in intel_iommu_iova_to_phys()