Lines Matching full:mm
11 #include <linux/mm.h>
47 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, in ptep_ipte_local() argument
54 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_local()
58 asce = asce ? : mm->context.asce; in ptep_ipte_local()
67 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, in ptep_ipte_global() argument
74 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_global()
78 asce = asce ? : mm->context.asce; in ptep_ipte_global()
87 static inline pte_t ptep_flush_direct(struct mm_struct *mm, in ptep_flush_direct() argument
96 atomic_inc(&mm->context.flush_count); in ptep_flush_direct()
98 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_flush_direct()
99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
102 atomic_dec(&mm->context.flush_count); in ptep_flush_direct()
106 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, in ptep_flush_lazy() argument
115 atomic_inc(&mm->context.flush_count); in ptep_flush_lazy()
116 if (cpumask_equal(&mm->context.cpu_attach_mask, in ptep_flush_lazy()
119 mm->context.flush_mm = 1; in ptep_flush_lazy()
121 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_lazy()
122 atomic_dec(&mm->context.flush_count); in ptep_flush_lazy()
174 struct mm_struct *mm) in pgste_update_all() argument
179 if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) in pgste_update_all()
195 struct mm_struct *mm) in pgste_set_key() argument
201 if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) in pgste_set_key()
239 static inline pgste_t pgste_pte_notify(struct mm_struct *mm, in pgste_pte_notify() argument
249 ptep_notify(mm, addr, ptep, bits); in pgste_pte_notify()
255 static inline pgste_t ptep_xchg_start(struct mm_struct *mm, in ptep_xchg_start() argument
260 if (mm_has_pgste(mm)) { in ptep_xchg_start()
262 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_xchg_start()
267 static inline pte_t ptep_xchg_commit(struct mm_struct *mm, in ptep_xchg_commit() argument
271 if (mm_has_pgste(mm)) { in ptep_xchg_commit()
273 pgste_set_key(ptep, pgste, new, mm); in ptep_xchg_commit()
275 pgste = pgste_update_all(old, pgste, mm); in ptep_xchg_commit()
288 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, in ptep_xchg_direct() argument
296 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_direct()
298 old = ptep_flush_direct(mm, addr, ptep, nodat); in ptep_xchg_direct()
299 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_direct()
309 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep, in ptep_reset_dat_prot() argument
313 atomic_inc(&mm->context.flush_count); in ptep_reset_dat_prot()
314 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_reset_dat_prot()
325 atomic_dec(&mm->context.flush_count); in ptep_reset_dat_prot()
330 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, in ptep_xchg_lazy() argument
338 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_lazy()
340 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_xchg_lazy()
341 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_lazy()
353 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_start() local
356 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_modify_prot_start()
358 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_modify_prot_start()
359 if (mm_has_pgste(mm)) { in ptep_modify_prot_start()
360 pgste = pgste_update_all(old, pgste, mm); in ptep_modify_prot_start()
370 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_commit() local
374 if (mm_has_pgste(mm)) { in ptep_modify_prot_commit()
376 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
385 static inline void pmdp_idte_local(struct mm_struct *mm, in pmdp_idte_local() argument
390 mm->context.asce, IDTE_LOCAL); in pmdp_idte_local()
393 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_local()
394 gmap_pmdp_idte_local(mm, addr); in pmdp_idte_local()
397 static inline void pmdp_idte_global(struct mm_struct *mm, in pmdp_idte_global() argument
402 mm->context.asce, IDTE_GLOBAL); in pmdp_idte_global()
403 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
404 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
407 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
408 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
411 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
412 gmap_pmdp_csp(mm, addr); in pmdp_idte_global()
416 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, in pmdp_flush_direct() argument
424 atomic_inc(&mm->context.flush_count); in pmdp_flush_direct()
426 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pmdp_flush_direct()
427 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
429 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_direct()
430 atomic_dec(&mm->context.flush_count); in pmdp_flush_direct()
434 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, in pmdp_flush_lazy() argument
442 atomic_inc(&mm->context.flush_count); in pmdp_flush_lazy()
443 if (cpumask_equal(&mm->context.cpu_attach_mask, in pmdp_flush_lazy()
446 mm->context.flush_mm = 1; in pmdp_flush_lazy()
447 if (mm_has_pgste(mm)) in pmdp_flush_lazy()
448 gmap_pmdp_invalidate(mm, addr); in pmdp_flush_lazy()
450 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_lazy()
452 atomic_dec(&mm->context.flush_count); in pmdp_flush_lazy()
457 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp) in pmd_lookup() argument
465 vma = vma_lookup(mm, addr); in pmd_lookup()
469 pgd = pgd_offset(mm, addr); in pmd_lookup()
490 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_direct() argument
496 old = pmdp_flush_direct(mm, addr, pmdp); in pmdp_xchg_direct()
503 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_lazy() argument
509 old = pmdp_flush_lazy(mm, addr, pmdp); in pmdp_xchg_lazy()
516 static inline void pudp_idte_local(struct mm_struct *mm, in pudp_idte_local() argument
521 mm->context.asce, IDTE_LOCAL); in pudp_idte_local()
526 static inline void pudp_idte_global(struct mm_struct *mm, in pudp_idte_global() argument
531 mm->context.asce, IDTE_GLOBAL); in pudp_idte_global()
542 static inline pud_t pudp_flush_direct(struct mm_struct *mm, in pudp_flush_direct() argument
550 atomic_inc(&mm->context.flush_count); in pudp_flush_direct()
552 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pudp_flush_direct()
553 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct()
555 pudp_idte_global(mm, addr, pudp); in pudp_flush_direct()
556 atomic_dec(&mm->context.flush_count); in pudp_flush_direct()
560 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pudp_xchg_direct() argument
566 old = pudp_flush_direct(mm, addr, pudp); in pudp_xchg_direct()
574 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
579 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_deposit()
582 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
585 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
586 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
589 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) in pgtable_trans_huge_withdraw() argument
595 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_withdraw()
598 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
601 pmd_huge_pte(mm, pmdp) = NULL; in pgtable_trans_huge_withdraw()
603 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; in pgtable_trans_huge_withdraw()
615 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, in ptep_set_pte_at() argument
624 pgste_set_key(ptep, pgste, entry, mm); in ptep_set_pte_at()
630 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_set_notify() argument
643 * @mm: pointer to the process mm_struct
652 int ptep_force_prot(struct mm_struct *mm, unsigned long addr, in ptep_force_prot() argument
672 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
673 pgste = pgste_update_all(entry, pgste, mm); in ptep_force_prot()
677 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
687 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, in ptep_shadow_pte() argument
714 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) in ptep_unshadow_pte() argument
722 ptep_flush_direct(mm, saddr, ptep, nodat); in ptep_unshadow_pte()
728 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) in ptep_zap_swap_entry() argument
731 dec_mm_counter(mm, MM_SWAPENTS); in ptep_zap_swap_entry()
735 dec_mm_counter(mm, mm_counter(page)); in ptep_zap_swap_entry()
740 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, in ptep_zap_unused() argument
755 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); in ptep_zap_unused()
756 pte_clear(mm, addr, ptep); in ptep_zap_unused()
764 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_zap_key() argument
784 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, in ptep_test_and_clear_uc() argument
797 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_uc()
799 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_test_and_clear_uc()
811 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in set_guest_storage_key() argument
824 switch (pmd_lookup(mm, addr, &pmdp)) { in set_guest_storage_key()
833 ptl = pmd_lock(mm, pmdp); in set_guest_storage_key()
852 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in set_guest_storage_key()
891 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in cond_set_guest_storage_key() argument
900 rc = get_guest_storage_key(current->mm, addr, &tmp); in cond_set_guest_storage_key()
912 rc = set_guest_storage_key(current->mm, addr, key, nq); in cond_set_guest_storage_key()
922 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) in reset_guest_reference_bit() argument
935 switch (pmd_lookup(mm, addr, &pmdp)) { in reset_guest_reference_bit()
944 ptl = pmd_lock(mm, pmdp); in reset_guest_reference_bit()
959 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in reset_guest_reference_bit()
984 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, in get_guest_storage_key() argument
999 switch (pmd_lookup(mm, addr, &pmdp)) { in get_guest_storage_key()
1008 ptl = pmd_lock(mm, pmdp); in get_guest_storage_key()
1023 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in get_guest_storage_key()
1041 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1051 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, in pgste_perform_essa() argument
1065 vma = vma_lookup(mm, hva); in pgste_perform_essa()
1068 ptep = get_locked_pte(mm, hva, &ptl); in pgste_perform_essa()
1149 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1157 int set_pgste_bits(struct mm_struct *mm, unsigned long hva, in set_pgste_bits() argument
1165 vma = vma_lookup(mm, hva); in set_pgste_bits()
1168 ptep = get_locked_pte(mm, hva, &ptl); in set_pgste_bits()
1184 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1190 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) in get_pgste() argument
1196 vma = vma_lookup(mm, hva); in get_pgste()
1199 ptep = get_locked_pte(mm, hva, &ptl); in get_pgste()