Lines Matching refs:gmap
35 static struct gmap *gmap_alloc(unsigned long limit) in gmap_alloc()
37 struct gmap *gmap; in gmap_alloc() local
59 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL); in gmap_alloc()
60 if (!gmap) in gmap_alloc()
62 INIT_LIST_HEAD(&gmap->crst_list); in gmap_alloc()
63 INIT_LIST_HEAD(&gmap->children); in gmap_alloc()
64 INIT_LIST_HEAD(&gmap->pt_list); in gmap_alloc()
65 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL); in gmap_alloc()
66 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC); in gmap_alloc()
67 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC); in gmap_alloc()
68 spin_lock_init(&gmap->guest_table_lock); in gmap_alloc()
69 spin_lock_init(&gmap->shadow_lock); in gmap_alloc()
70 refcount_set(&gmap->ref_count, 1); in gmap_alloc()
75 list_add(&page->lru, &gmap->crst_list); in gmap_alloc()
78 gmap->table = table; in gmap_alloc()
79 gmap->asce = atype | _ASCE_TABLE_LENGTH | in gmap_alloc()
81 gmap->asce_end = limit; in gmap_alloc()
82 return gmap; in gmap_alloc()
85 kfree(gmap); in gmap_alloc()
97 struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) in gmap_create()
99 struct gmap *gmap; in gmap_create() local
102 gmap = gmap_alloc(limit); in gmap_create()
103 if (!gmap) in gmap_create()
105 gmap->mm = mm; in gmap_create()
107 list_add_rcu(&gmap->list, &mm->context.gmap_list); in gmap_create()
109 gmap_asce = gmap->asce; in gmap_create()
114 return gmap; in gmap_create()
118 static void gmap_flush_tlb(struct gmap *gmap) in gmap_flush_tlb() argument
121 __tlb_flush_idte(gmap->asce); in gmap_flush_tlb()
183 static void gmap_free(struct gmap *gmap) in gmap_free() argument
188 if (!(gmap_is_shadow(gmap) && gmap->removed)) in gmap_free()
189 gmap_flush_tlb(gmap); in gmap_free()
191 list_for_each_entry_safe(page, next, &gmap->crst_list, lru) in gmap_free()
193 gmap_radix_tree_free(&gmap->guest_to_host); in gmap_free()
194 gmap_radix_tree_free(&gmap->host_to_guest); in gmap_free()
197 if (gmap_is_shadow(gmap)) { in gmap_free()
199 list_for_each_entry_safe(page, next, &gmap->pt_list, lru) in gmap_free()
201 gmap_rmap_radix_tree_free(&gmap->host_to_rmap); in gmap_free()
203 gmap_put(gmap->parent); in gmap_free()
206 kfree(gmap); in gmap_free()
215 struct gmap *gmap_get(struct gmap *gmap) in gmap_get() argument
217 refcount_inc(&gmap->ref_count); in gmap_get()
218 return gmap; in gmap_get()
228 void gmap_put(struct gmap *gmap) in gmap_put() argument
230 if (refcount_dec_and_test(&gmap->ref_count)) in gmap_put()
231 gmap_free(gmap); in gmap_put()
239 void gmap_remove(struct gmap *gmap) in gmap_remove() argument
241 struct gmap *sg, *next; in gmap_remove()
245 if (!list_empty(&gmap->children)) { in gmap_remove()
246 spin_lock(&gmap->shadow_lock); in gmap_remove()
247 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
251 spin_unlock(&gmap->shadow_lock); in gmap_remove()
254 spin_lock(&gmap->mm->context.lock); in gmap_remove()
255 list_del_rcu(&gmap->list); in gmap_remove()
256 if (list_empty(&gmap->mm->context.gmap_list)) in gmap_remove()
258 else if (list_is_singular(&gmap->mm->context.gmap_list)) in gmap_remove()
259 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, in gmap_remove()
260 struct gmap, list)->asce; in gmap_remove()
263 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); in gmap_remove()
264 spin_unlock(&gmap->mm->context.lock); in gmap_remove()
267 gmap_put(gmap); in gmap_remove()
275 void gmap_enable(struct gmap *gmap) in gmap_enable() argument
277 S390_lowcore.gmap = (unsigned long) gmap; in gmap_enable()
285 void gmap_disable(struct gmap *gmap) in gmap_disable() argument
287 S390_lowcore.gmap = 0UL; in gmap_disable()
296 struct gmap *gmap_get_enabled(void) in gmap_get_enabled()
298 return (struct gmap *) S390_lowcore.gmap; in gmap_get_enabled()
305 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, in gmap_alloc_table() argument
317 spin_lock(&gmap->guest_table_lock); in gmap_alloc_table()
319 list_add(&page->lru, &gmap->crst_list); in gmap_alloc_table()
325 spin_unlock(&gmap->guest_table_lock); in gmap_alloc_table()
356 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) in __gmap_unlink_by_vmaddr() argument
361 BUG_ON(gmap_is_shadow(gmap)); in __gmap_unlink_by_vmaddr()
362 spin_lock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
363 entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in __gmap_unlink_by_vmaddr()
368 spin_unlock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
379 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr) in __gmap_unmap_by_gaddr() argument
383 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, in __gmap_unmap_by_gaddr()
385 return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0; in __gmap_unmap_by_gaddr()
396 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) in gmap_unmap_segment() argument
401 BUG_ON(gmap_is_shadow(gmap)); in gmap_unmap_segment()
408 down_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
410 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_unmap_segment()
411 up_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
413 gmap_flush_tlb(gmap); in gmap_unmap_segment()
427 int gmap_map_segment(struct gmap *gmap, unsigned long from, in gmap_map_segment() argument
433 BUG_ON(gmap_is_shadow(gmap)); in gmap_map_segment()
437 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) in gmap_map_segment()
441 down_write(&gmap->mm->mmap_sem); in gmap_map_segment()
444 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_map_segment()
446 if (radix_tree_insert(&gmap->guest_to_host, in gmap_map_segment()
451 up_write(&gmap->mm->mmap_sem); in gmap_map_segment()
453 gmap_flush_tlb(gmap); in gmap_map_segment()
456 gmap_unmap_segment(gmap, to, len); in gmap_map_segment()
474 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr) in __gmap_translate() argument
479 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); in __gmap_translate()
494 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr) in gmap_translate() argument
498 down_read(&gmap->mm->mmap_sem); in gmap_translate()
499 rc = __gmap_translate(gmap, gaddr); in gmap_translate()
500 up_read(&gmap->mm->mmap_sem); in gmap_translate()
514 struct gmap *gmap; in gmap_unlink() local
518 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_unlink()
519 flush = __gmap_unlink_by_vmaddr(gmap, vmaddr); in gmap_unlink()
521 gmap_flush_tlb(gmap); in gmap_unlink()
526 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new,
540 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) in __gmap_link() argument
552 BUG_ON(gmap_is_shadow(gmap)); in __gmap_link()
554 table = gmap->table; in __gmap_link()
555 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { in __gmap_link()
558 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, in __gmap_link()
563 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { in __gmap_link()
566 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, in __gmap_link()
571 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { in __gmap_link()
574 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, in __gmap_link()
581 mm = gmap->mm; in __gmap_link()
594 if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) in __gmap_link()
601 spin_lock(&gmap->guest_table_lock); in __gmap_link()
603 rc = radix_tree_insert(&gmap->host_to_guest, in __gmap_link()
619 gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr); in __gmap_link()
621 spin_unlock(&gmap->guest_table_lock); in __gmap_link()
636 int gmap_fault(struct gmap *gmap, unsigned long gaddr, in gmap_fault() argument
643 down_read(&gmap->mm->mmap_sem); in gmap_fault()
647 vmaddr = __gmap_translate(gmap, gaddr); in gmap_fault()
652 if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags, in gmap_fault()
664 rc = __gmap_link(gmap, gaddr, vmaddr); in gmap_fault()
666 up_read(&gmap->mm->mmap_sem); in gmap_fault()
674 void __gmap_zap(struct gmap *gmap, unsigned long gaddr) in __gmap_zap() argument
681 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, in __gmap_zap()
686 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl); in __gmap_zap()
688 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0); in __gmap_zap()
694 void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) in gmap_discard() argument
699 down_read(&gmap->mm->mmap_sem); in gmap_discard()
704 radix_tree_lookup(&gmap->guest_to_host, in gmap_discard()
710 vma = find_vma(gmap->mm, vmaddr); in gmap_discard()
722 up_read(&gmap->mm->mmap_sem); in gmap_discard()
760 static void gmap_call_notifier(struct gmap *gmap, unsigned long start, in gmap_call_notifier() argument
766 nb->notifier_call(gmap, start, end); in gmap_call_notifier()
787 static inline unsigned long *gmap_table_walk(struct gmap *gmap, in gmap_table_walk() argument
792 if ((gmap->asce & _ASCE_TYPE_MASK) + 4 < (level * 4)) in gmap_table_walk()
794 if (gmap_is_shadow(gmap) && gmap->removed) in gmap_table_walk()
796 if (gaddr & (-1UL << (31 + ((gmap->asce & _ASCE_TYPE_MASK) >> 2)*11))) in gmap_table_walk()
798 table = gmap->table; in gmap_table_walk()
799 switch (gmap->asce & _ASCE_TYPE_MASK) { in gmap_table_walk()
845 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_walk() argument
850 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_walk()
852 table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */ in gmap_pte_op_walk()
855 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); in gmap_pte_op_walk()
869 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_fixup() argument
872 struct mm_struct *mm = gmap->mm; in gmap_pte_op_fixup()
876 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_fixup()
884 return __gmap_link(gmap, gaddr, vmaddr); in gmap_pte_op_fixup()
905 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr) in gmap_pmd_op_walk() argument
909 BUG_ON(gmap_is_shadow(gmap)); in gmap_pmd_op_walk()
910 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); in gmap_pmd_op_walk()
915 if (!gmap->mm->context.allow_gmap_hpage_1m) in gmap_pmd_op_walk()
918 spin_lock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
920 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
926 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
935 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) in gmap_pmd_op_end() argument
938 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_end()
955 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pmd() argument
968 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
974 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
1000 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pte() argument
1011 ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); in gmap_protect_pte()
1018 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); in gmap_protect_pte()
1036 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, in gmap_protect_range() argument
1043 BUG_ON(gmap_is_shadow(gmap)); in gmap_protect_range()
1046 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_protect_range()
1049 rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1056 rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1064 gmap_pmd_op_end(gmap, pmdp); in gmap_protect_range()
1071 vmaddr = __gmap_translate(gmap, gaddr); in gmap_protect_range()
1074 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot); in gmap_protect_range()
1096 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, in gmap_mprotect_notify() argument
1101 if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap)) in gmap_mprotect_notify()
1105 down_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
1106 rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT); in gmap_mprotect_notify()
1107 up_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
1125 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) in gmap_read_table() argument
1132 if (gmap_is_shadow(gmap)) in gmap_read_table()
1137 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); in gmap_read_table()
1152 vmaddr = __gmap_translate(gmap, gaddr); in gmap_read_table()
1157 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ); in gmap_read_table()
1173 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap()
1201 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap()
1204 struct gmap *parent; in gmap_protect_rmap()
1282 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page()
1302 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt()
1319 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt()
1348 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt()
1376 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt()
1405 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t()
1433 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t()
1462 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t()
1490 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t()
1519 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t()
1549 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow()
1586 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, in gmap_find_shadow()
1589 struct gmap *sg; in gmap_find_shadow()
1615 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid()
1638 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, in gmap_shadow()
1641 struct gmap *sg, *new; in gmap_shadow()
1730 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t()
1814 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t()
1897 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt()
1981 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup()
2021 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt()
2100 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page()
2102 struct gmap *parent; in gmap_shadow_page()
2168 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify()
2235 struct gmap *gmap, *sg, *next; in ptep_notify() local
2240 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in ptep_notify()
2241 spin_lock(&gmap->guest_table_lock); in ptep_notify()
2242 table = radix_tree_lookup(&gmap->host_to_guest, in ptep_notify()
2246 spin_unlock(&gmap->guest_table_lock); in ptep_notify()
2250 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { in ptep_notify()
2251 spin_lock(&gmap->shadow_lock); in ptep_notify()
2253 &gmap->children, list) in ptep_notify()
2255 spin_unlock(&gmap->shadow_lock); in ptep_notify()
2258 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); in ptep_notify()
2264 static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp, in pmdp_notify_gmap() argument
2268 gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); in pmdp_notify_gmap()
2281 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, in gmap_pmdp_xchg() argument
2285 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_xchg()
2288 __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, in gmap_pmdp_xchg()
2301 struct gmap *gmap; in gmap_pmdp_clear() local
2305 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_clear()
2306 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2307 pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_clear()
2311 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_clear()
2318 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2354 struct gmap *gmap; in gmap_pmdp_idte_local() local
2358 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_local()
2359 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2360 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_local()
2365 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_local()
2370 gmap->asce, IDTE_LOCAL); in gmap_pmdp_idte_local()
2375 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2389 struct gmap *gmap; in gmap_pmdp_idte_global() local
2393 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_global()
2394 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2395 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_global()
2400 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_global()
2405 gmap->asce, IDTE_GLOBAL); in gmap_pmdp_idte_global()
2412 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2427 static bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp, in gmap_test_and_clear_dirty_pmd() argument
2440 gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0); in gmap_test_and_clear_dirty_pmd()
2454 void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4], in gmap_sync_dirty_log_pmd() argument
2462 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_sync_dirty_log_pmd()
2467 if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr)) in gmap_sync_dirty_log_pmd()
2471 ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); in gmap_sync_dirty_log_pmd()
2474 if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) in gmap_sync_dirty_log_pmd()
2479 gmap_pmd_op_end(gmap, pmdp); in gmap_sync_dirty_log_pmd()