Lines Matching refs:pte
645 pte_t pte, struct page *page) in print_bad_pte() argument
682 (long long)pte_val(pte), (long long)pmd_val(*pmd)); in print_bad_pte()
749 pte_t pte) in vm_normal_page() argument
751 unsigned long pfn = pte_pfn(pte); in vm_normal_page()
754 if (likely(!pte_special(pte))) in vm_normal_page()
759 print_bad_pte(vma, addr, pte, NULL); in vm_normal_page()
784 print_bad_pte(vma, addr, pte, NULL); in vm_normal_page()
808 pte_t pte = *src_pte; in copy_one_pte() local
812 if (unlikely(!pte_present(pte))) { in copy_one_pte()
813 if (!pte_file(pte)) { in copy_one_pte()
814 swp_entry_t entry = pte_to_swp_entry(pte); in copy_one_pte()
844 pte = swp_entry_to_pte(entry); in copy_one_pte()
846 pte = pte_swp_mksoft_dirty(pte); in copy_one_pte()
847 set_pte_at(src_mm, addr, src_pte, pte); in copy_one_pte()
860 pte = pte_wrprotect(pte); in copy_one_pte()
868 pte = pte_mkclean(pte); in copy_one_pte()
869 pte = pte_mkold(pte); in copy_one_pte()
871 page = vm_normal_page(vma, addr, pte); in copy_one_pte()
882 set_pte_at(dst_mm, addr, dst_pte, pte); in copy_one_pte()
1083 pte_t *pte; in zap_pte_range() local
1088 pte = start_pte; in zap_pte_range()
1091 pte_t ptent = *pte; in zap_pte_range()
1118 ptent = ptep_get_and_clear_full(mm, addr, pte, in zap_pte_range()
1120 tlb_remove_tlb_entry(tlb, pte, addr); in zap_pte_range()
1129 set_pte_at(mm, addr, pte, ptfile); in zap_pte_range()
1180 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); in zap_pte_range()
1181 } while (pte++, addr += PAGE_SIZE, addr != end); in zap_pte_range()
1469 pte_t *pte; in insert_page() local
1477 pte = get_locked_pte(mm, addr, &ptl); in insert_page()
1478 if (!pte) in insert_page()
1481 if (!pte_none(*pte)) in insert_page()
1488 set_pte_at(mm, addr, pte, mk_pte(page, prot)); in insert_page()
1491 pte_unmap_unlock(pte, ptl); in insert_page()
1494 pte_unmap_unlock(pte, ptl); in insert_page()
1547 pte_t *pte, entry; in insert_pfn() local
1551 pte = get_locked_pte(mm, addr, &ptl); in insert_pfn()
1552 if (!pte) in insert_pfn()
1555 if (!pte_none(*pte)) in insert_pfn()
1560 set_pte_at(mm, addr, pte, entry); in insert_pfn()
1561 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ in insert_pfn()
1565 pte_unmap_unlock(pte, ptl); in insert_pfn()
1649 pte_t *pte; in remap_pte_range() local
1652 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); in remap_pte_range()
1653 if (!pte) in remap_pte_range()
1657 BUG_ON(!pte_none(*pte)); in remap_pte_range()
1658 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); in remap_pte_range()
1660 } while (pte++, addr += PAGE_SIZE, addr != end); in remap_pte_range()
1662 pte_unmap_unlock(pte - 1, ptl); in remap_pte_range()
1826 pte_t *pte; in apply_to_pte_range() local
1831 pte = (mm == &init_mm) ? in apply_to_pte_range()
1834 if (!pte) in apply_to_pte_range()
1844 err = fn(pte++, token, addr, data); in apply_to_pte_range()
1852 pte_unmap_unlock(pte-1, ptl); in apply_to_pte_range()
2412 pte_t pte; in do_swap_page() local
2518 pte = mk_pte(page, vma->vm_page_prot); in do_swap_page()
2520 pte = maybe_mkwrite(pte_mkdirty(pte), vma); in do_swap_page()
2527 pte = pte_mksoft_dirty(pte); in do_swap_page()
2528 set_pte_at(mm, address, page_table, pte); in do_swap_page()
2556 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); in do_swap_page()
2710 struct page *page, pte_t *pte, bool write, bool anon) in do_set_pte() argument
2718 else if (pte_file(*pte) && pte_file_soft_dirty(*pte)) in do_set_pte()
2727 set_pte_at(vma->vm_mm, address, pte, entry); in do_set_pte()
2730 update_mmu_cache(vma, address, pte); in do_set_pte()
2798 pte_t *pte, pgoff_t pgoff, unsigned int flags) in do_fault_around() argument
2810 pte -= off; in do_fault_around()
2823 while (!pte_none(*pte)) { in do_fault_around()
2829 pte++; in do_fault_around()
2833 vmf.pte = pte; in do_fault_around()
2846 pte_t *pte; in do_read_fault() local
2856 pte = pte_offset_map_lock(mm, pmd, address, &ptl); in do_read_fault()
2857 do_fault_around(vma, address, pte, pgoff, flags); in do_read_fault()
2858 if (!pte_same(*pte, orig_pte)) in do_read_fault()
2860 pte_unmap_unlock(pte, ptl); in do_read_fault()
2867 pte = pte_offset_map_lock(mm, pmd, address, &ptl); in do_read_fault()
2868 if (unlikely(!pte_same(*pte, orig_pte))) { in do_read_fault()
2869 pte_unmap_unlock(pte, ptl); in do_read_fault()
2874 do_set_pte(vma, address, fault_page, pte, false, false); in do_read_fault()
2877 pte_unmap_unlock(pte, ptl); in do_read_fault()
2888 pte_t *pte; in do_cow_fault() local
2910 pte = pte_offset_map_lock(mm, pmd, address, &ptl); in do_cow_fault()
2911 if (unlikely(!pte_same(*pte, orig_pte))) { in do_cow_fault()
2912 pte_unmap_unlock(pte, ptl); in do_cow_fault()
2917 do_set_pte(vma, address, new_page, pte, true, true); in do_cow_fault()
2920 pte_unmap_unlock(pte, ptl); in do_cow_fault()
2937 pte_t *pte; in do_shared_fault() local
2959 pte = pte_offset_map_lock(mm, pmd, address, &ptl); in do_shared_fault()
2960 if (unlikely(!pte_same(*pte, orig_pte))) { in do_shared_fault()
2961 pte_unmap_unlock(pte, ptl); in do_shared_fault()
2966 do_set_pte(vma, address, fault_page, pte, true, false); in do_shared_fault()
2967 pte_unmap_unlock(pte, ptl); in do_shared_fault()
3070 unsigned long addr, pte_t pte, pte_t *ptep, pmd_t *pmd) in do_numa_page() argument
3091 if (unlikely(!pte_same(*ptep, pte))) { in do_numa_page()
3096 pte = pte_mknonnuma(pte); in do_numa_page()
3097 set_pte_at(mm, addr, ptep, pte); in do_numa_page()
3100 page = vm_normal_page(vma, addr, pte); in do_numa_page()
3112 if (!pte_write(pte)) in do_numa_page()
3162 pte_t *pte, pmd_t *pmd, unsigned int flags) in handle_pte_fault() argument
3167 entry = ACCESS_ONCE(*pte); in handle_pte_fault()
3172 pte, pmd, flags, entry); in handle_pte_fault()
3174 pte, pmd, flags); in handle_pte_fault()
3178 pte, pmd, flags, entry); in handle_pte_fault()
3180 pte, pmd, flags, entry); in handle_pte_fault()
3184 return do_numa_page(mm, vma, address, entry, pte, pmd); in handle_pte_fault()
3188 if (unlikely(!pte_same(*pte, entry))) in handle_pte_fault()
3193 pte, pmd, ptl, entry); in handle_pte_fault()
3197 if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) { in handle_pte_fault()
3198 update_mmu_cache(vma, address, pte); in handle_pte_fault()
3210 pte_unmap_unlock(pte, ptl); in handle_pte_fault()
3226 pte_t *pte; in __handle_mm_fault() local
3305 pte = pte_offset_map(pmd, address); in __handle_mm_fault()
3307 return handle_pte_fault(mm, vma, address, pte, pmd, flags); in __handle_mm_fault()
3490 pte_t *ptep, pte; in follow_phys() local
3498 pte = *ptep; in follow_phys()
3500 if ((flags & FOLL_WRITE) && !pte_write(pte)) in follow_phys()
3503 *prot = pgprot_val(pte_pgprot(pte)); in follow_phys()
3504 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; in follow_phys()