Lines Matching +full:mm +full:- +full:0
1 // SPDX-License-Identifier: GPL-2.0
11 #include <linux/mm.h>
25 #include <asm/page-states.h>
47 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, in ptep_ipte_local() argument
53 opt = 0; in ptep_ipte_local()
54 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_local()
55 if (asce == 0UL || nodat) in ptep_ipte_local()
57 if (asce != -1UL) { in ptep_ipte_local()
58 asce = asce ? : mm->context.asce; in ptep_ipte_local()
63 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
67 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, in ptep_ipte_global() argument
73 opt = 0; in ptep_ipte_global()
74 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_global()
75 if (asce == 0UL || nodat) in ptep_ipte_global()
77 if (asce != -1UL) { in ptep_ipte_global()
78 asce = asce ? : mm->context.asce; in ptep_ipte_global()
83 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
87 static inline pte_t ptep_flush_direct(struct mm_struct *mm, in ptep_flush_direct() argument
96 atomic_inc(&mm->context.flush_count); in ptep_flush_direct()
98 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_flush_direct()
99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
102 atomic_dec(&mm->context.flush_count); in ptep_flush_direct()
106 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, in ptep_flush_lazy() argument
115 atomic_inc(&mm->context.flush_count); in ptep_flush_lazy()
116 if (cpumask_equal(&mm->context.cpu_attach_mask, in ptep_flush_lazy()
119 mm->context.flush_mm = 1; in ptep_flush_lazy()
121 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_lazy()
122 atomic_dec(&mm->context.flush_count); in ptep_flush_lazy()
128 unsigned long new = 0; in pgste_get_lock()
133 " lg %0,%2\n" in pgste_get_lock()
134 "0: lgr %1,%0\n" in pgste_get_lock()
135 " nihh %0,0xff7f\n" /* clear PCL bit in old */ in pgste_get_lock()
136 " oihh %1,0x0080\n" /* set PCL bit in new */ in pgste_get_lock()
137 " csg %0,%1,%2\n" in pgste_get_lock()
138 " jl 0b\n" in pgste_get_lock()
149 " nihh %1,0xff7f\n" /* clear PCL bit */ in pgste_set_unlock()
150 " stg %1,%0\n" in pgste_set_unlock()
159 unsigned long pgste = 0; in pgste_get()
174 struct mm_struct *mm) in pgste_update_all() argument
179 if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) in pgste_update_all()
195 struct mm_struct *mm) in pgste_set_key() argument
201 if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) in pgste_set_key()
208 * key C/R to 0. in pgste_set_key()
212 page_set_storage_key(address, nkey, 0); in pgste_set_key()
224 * Without enhanced suppression-on-protection force in pgste_set_pte()
231 /* This pte allows write access, set user-dirty */ in pgste_set_pte()
239 static inline pgste_t pgste_pte_notify(struct mm_struct *mm, in pgste_pte_notify() argument
249 ptep_notify(mm, addr, ptep, bits); in pgste_pte_notify()
255 static inline pgste_t ptep_xchg_start(struct mm_struct *mm, in ptep_xchg_start() argument
258 pgste_t pgste = __pgste(0); in ptep_xchg_start()
260 if (mm_has_pgste(mm)) { in ptep_xchg_start()
262 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_xchg_start()
267 static inline pte_t ptep_xchg_commit(struct mm_struct *mm, in ptep_xchg_commit() argument
271 if (mm_has_pgste(mm)) { in ptep_xchg_commit()
273 pgste_set_key(ptep, pgste, new, mm); in ptep_xchg_commit()
275 pgste = pgste_update_all(old, pgste, mm); in ptep_xchg_commit()
288 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, in ptep_xchg_direct() argument
296 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_direct()
298 old = ptep_flush_direct(mm, addr, ptep, nodat); in ptep_xchg_direct()
299 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_direct()
309 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep, in ptep_reset_dat_prot() argument
313 atomic_inc(&mm->context.flush_count); in ptep_reset_dat_prot()
314 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_reset_dat_prot()
315 __ptep_rdp(addr, ptep, 0, 0, 1); in ptep_reset_dat_prot()
317 __ptep_rdp(addr, ptep, 0, 0, 0); in ptep_reset_dat_prot()
325 atomic_dec(&mm->context.flush_count); in ptep_reset_dat_prot()
330 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, in ptep_xchg_lazy() argument
338 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_lazy()
340 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_xchg_lazy()
341 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_lazy()
353 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_start() local
356 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_modify_prot_start()
358 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_modify_prot_start()
359 if (mm_has_pgste(mm)) { in ptep_modify_prot_start()
360 pgste = pgste_update_all(old, pgste, mm); in ptep_modify_prot_start()
370 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_commit() local
374 if (mm_has_pgste(mm)) { in ptep_modify_prot_commit()
376 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
385 static inline void pmdp_idte_local(struct mm_struct *mm, in pmdp_idte_local() argument
390 mm->context.asce, IDTE_LOCAL); in pmdp_idte_local()
392 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); in pmdp_idte_local()
393 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_local()
394 gmap_pmdp_idte_local(mm, addr); in pmdp_idte_local()
397 static inline void pmdp_idte_global(struct mm_struct *mm, in pmdp_idte_global() argument
402 mm->context.asce, IDTE_GLOBAL); in pmdp_idte_global()
403 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
404 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
406 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); in pmdp_idte_global()
407 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
408 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
411 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
412 gmap_pmdp_csp(mm, addr); in pmdp_idte_global()
416 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, in pmdp_flush_direct() argument
424 atomic_inc(&mm->context.flush_count); in pmdp_flush_direct()
426 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pmdp_flush_direct()
427 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
429 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_direct()
430 atomic_dec(&mm->context.flush_count); in pmdp_flush_direct()
434 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, in pmdp_flush_lazy() argument
442 atomic_inc(&mm->context.flush_count); in pmdp_flush_lazy()
443 if (cpumask_equal(&mm->context.cpu_attach_mask, in pmdp_flush_lazy()
446 mm->context.flush_mm = 1; in pmdp_flush_lazy()
447 if (mm_has_pgste(mm)) in pmdp_flush_lazy()
448 gmap_pmdp_invalidate(mm, addr); in pmdp_flush_lazy()
450 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_lazy()
452 atomic_dec(&mm->context.flush_count); in pmdp_flush_lazy()
457 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp) in pmd_lookup() argument
465 vma = vma_lookup(mm, addr); in pmd_lookup()
467 return -EFAULT; in pmd_lookup()
469 pgd = pgd_offset(mm, addr); in pmd_lookup()
471 return -ENOENT; in pmd_lookup()
475 return -ENOENT; in pmd_lookup()
479 return -ENOENT; in pmd_lookup()
483 return -EFAULT; in pmd_lookup()
486 return 0; in pmd_lookup()
490 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_direct() argument
496 old = pmdp_flush_direct(mm, addr, pmdp); in pmdp_xchg_direct()
503 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_lazy() argument
509 old = pmdp_flush_lazy(mm, addr, pmdp); in pmdp_xchg_lazy()
516 static inline void pudp_idte_local(struct mm_struct *mm, in pudp_idte_local() argument
521 mm->context.asce, IDTE_LOCAL); in pudp_idte_local()
523 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); in pudp_idte_local()
526 static inline void pudp_idte_global(struct mm_struct *mm, in pudp_idte_global() argument
531 mm->context.asce, IDTE_GLOBAL); in pudp_idte_global()
533 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); in pudp_idte_global()
537 * re-use _pmd_csp() here in pudp_idte_global()
542 static inline pud_t pudp_flush_direct(struct mm_struct *mm, in pudp_flush_direct() argument
550 atomic_inc(&mm->context.flush_count); in pudp_flush_direct()
552 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pudp_flush_direct()
553 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct()
555 pudp_idte_global(mm, addr, pudp); in pudp_flush_direct()
556 atomic_dec(&mm->context.flush_count); in pudp_flush_direct()
560 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pudp_xchg_direct() argument
566 old = pudp_flush_direct(mm, addr, pudp); in pudp_xchg_direct()
574 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
579 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_deposit()
582 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
585 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
586 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
589 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) in pgtable_trans_huge_withdraw() argument
595 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_withdraw()
598 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
601 pmd_huge_pte(mm, pmdp) = NULL; in pgtable_trans_huge_withdraw()
603 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; in pgtable_trans_huge_withdraw()
615 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, in ptep_set_pte_at() argument
624 pgste_set_key(ptep, pgste, entry, mm); in ptep_set_pte_at()
630 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_set_notify() argument
642 * ptep_force_prot - change access rights of a locked pte
643 * @mm: pointer to the process mm_struct
649 * Returns 0 if the access rights were changed and -EAGAIN if the current
652 int ptep_force_prot(struct mm_struct *mm, unsigned long addr, in ptep_force_prot() argument
667 return -EAGAIN; in ptep_force_prot()
672 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
673 pgste = pgste_update_all(entry, pgste, mm); in ptep_force_prot()
677 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
684 return 0; in ptep_force_prot()
687 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, in ptep_shadow_pte() argument
692 int rc = -EAGAIN; in ptep_shadow_pte()
695 return 0; /* already shadowed */ in ptep_shadow_pte()
705 /* don't touch the storage key - it belongs to parent pgste */ in ptep_shadow_pte()
714 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) in ptep_unshadow_pte() argument
722 ptep_flush_direct(mm, saddr, ptep, nodat); in ptep_unshadow_pte()
723 /* don't touch the storage key - it belongs to parent pgste */ in ptep_unshadow_pte()
728 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) in ptep_zap_swap_entry() argument
731 dec_mm_counter(mm, MM_SWAPENTS); in ptep_zap_swap_entry()
735 dec_mm_counter(mm, mm_counter(page)); in ptep_zap_swap_entry()
740 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, in ptep_zap_unused() argument
747 /* Zap unused and logically-zero pages */ in ptep_zap_unused()
755 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); in ptep_zap_unused()
756 pte_clear(mm, addr, ptep); in ptep_zap_unused()
764 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_zap_key() argument
776 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0); in ptep_zap_key()
784 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, in ptep_test_and_clear_uc() argument
797 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_uc()
799 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_test_and_clear_uc()
811 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in set_guest_storage_key() argument
822 * we can ignore attempts to set the key to 0, because it already is 0. in set_guest_storage_key()
824 switch (pmd_lookup(mm, addr, &pmdp)) { in set_guest_storage_key()
825 case -ENOENT: in set_guest_storage_key()
826 return key ? -EFAULT : 0; in set_guest_storage_key()
827 case 0: in set_guest_storage_key()
830 return -EFAULT; in set_guest_storage_key()
833 ptl = pmd_lock(mm, pmdp); in set_guest_storage_key()
836 return key ? -EFAULT : 0; in set_guest_storage_key()
848 return 0; in set_guest_storage_key()
852 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in set_guest_storage_key()
880 return 0; in set_guest_storage_key()
888 * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
889 * storage key was updated and -EFAULT on access errors.
891 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in cond_set_guest_storage_key() argument
900 rc = get_guest_storage_key(current->mm, addr, &tmp); in cond_set_guest_storage_key()
910 return 0; in cond_set_guest_storage_key()
912 rc = set_guest_storage_key(current->mm, addr, key, nq); in cond_set_guest_storage_key()
913 return rc < 0 ? rc : 1; in cond_set_guest_storage_key()
920 * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
922 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) in reset_guest_reference_bit() argument
929 int cc = 0; in reset_guest_reference_bit()
933 * the storage key is 0 and there is nothing for us to do. in reset_guest_reference_bit()
935 switch (pmd_lookup(mm, addr, &pmdp)) { in reset_guest_reference_bit()
936 case -ENOENT: in reset_guest_reference_bit()
937 return 0; in reset_guest_reference_bit()
938 case 0: in reset_guest_reference_bit()
941 return -EFAULT; in reset_guest_reference_bit()
944 ptl = pmd_lock(mm, pmdp); in reset_guest_reference_bit()
947 return 0; in reset_guest_reference_bit()
959 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in reset_guest_reference_bit()
969 /* Merge real referenced bit into host-set */ in reset_guest_reference_bit()
984 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, in get_guest_storage_key() argument
995 * the storage key is 0. in get_guest_storage_key()
997 *key = 0; in get_guest_storage_key()
999 switch (pmd_lookup(mm, addr, &pmdp)) { in get_guest_storage_key()
1000 case -ENOENT: in get_guest_storage_key()
1001 return 0; in get_guest_storage_key()
1002 case 0: in get_guest_storage_key()
1005 return -EFAULT; in get_guest_storage_key()
1008 ptl = pmd_lock(mm, pmdp); in get_guest_storage_key()
1011 return 0; in get_guest_storage_key()
1019 return 0; in get_guest_storage_key()
1023 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in get_guest_storage_key()
1035 return 0; in get_guest_storage_key()
1040 * pgste_perform_essa - perform ESSA actions on the PGSTE.
1041 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1047 * Return: 1 if the page is to be added to the CBRL, otherwise 0,
1048 * or < 0 in case of error. -EINVAL is returned for invalid values
1049 * of orc, -EFAULT for invalid addresses.
1051 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, in pgste_perform_essa() argument
1059 int res = 0; in pgste_perform_essa()
1063 return -EINVAL; in pgste_perform_essa()
1065 vma = vma_lookup(mm, hva); in pgste_perform_essa()
1067 return -EFAULT; in pgste_perform_essa()
1068 ptep = get_locked_pte(mm, hva, &ptl); in pgste_perform_essa()
1070 return -EFAULT; in pgste_perform_essa()
1148 * set_pgste_bits - set specific PGSTE bits.
1149 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1155 * Return: 0 on success, < 0 in case of error.
1157 int set_pgste_bits(struct mm_struct *mm, unsigned long hva, in set_pgste_bits() argument
1165 vma = vma_lookup(mm, hva); in set_pgste_bits()
1167 return -EFAULT; in set_pgste_bits()
1168 ptep = get_locked_pte(mm, hva, &ptl); in set_pgste_bits()
1170 return -EFAULT; in set_pgste_bits()
1178 return 0; in set_pgste_bits()
1183 * get_pgste - get the current PGSTE for the given address.
1184 * @mm: the memory context. It must have PGSTEs, no check is performed here!
1188 * Return: 0 on success, < 0 in case of error.
1190 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) in get_pgste() argument
1196 vma = vma_lookup(mm, hva); in get_pgste()
1198 return -EFAULT; in get_pgste()
1199 ptep = get_locked_pte(mm, hva, &ptl); in get_pgste()
1201 return -EFAULT; in get_pgste()
1204 return 0; in get_pgste()