Lines Matching refs:gmap
32 static struct gmap *gmap_alloc(unsigned long limit) in gmap_alloc()
34 struct gmap *gmap; in gmap_alloc() local
56 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL); in gmap_alloc()
57 if (!gmap) in gmap_alloc()
59 INIT_LIST_HEAD(&gmap->crst_list); in gmap_alloc()
60 INIT_LIST_HEAD(&gmap->children); in gmap_alloc()
61 INIT_LIST_HEAD(&gmap->pt_list); in gmap_alloc()
62 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL); in gmap_alloc()
63 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC); in gmap_alloc()
64 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC); in gmap_alloc()
65 spin_lock_init(&gmap->guest_table_lock); in gmap_alloc()
66 spin_lock_init(&gmap->shadow_lock); in gmap_alloc()
67 atomic_set(&gmap->ref_count, 1); in gmap_alloc()
72 list_add(&page->lru, &gmap->crst_list); in gmap_alloc()
75 gmap->table = table; in gmap_alloc()
76 gmap->asce = atype | _ASCE_TABLE_LENGTH | in gmap_alloc()
78 gmap->asce_end = limit; in gmap_alloc()
79 return gmap; in gmap_alloc()
82 kfree(gmap); in gmap_alloc()
94 struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) in gmap_create()
96 struct gmap *gmap; in gmap_create() local
99 gmap = gmap_alloc(limit); in gmap_create()
100 if (!gmap) in gmap_create()
102 gmap->mm = mm; in gmap_create()
104 list_add_rcu(&gmap->list, &mm->context.gmap_list); in gmap_create()
106 gmap_asce = gmap->asce; in gmap_create()
111 return gmap; in gmap_create()
115 static void gmap_flush_tlb(struct gmap *gmap) in gmap_flush_tlb() argument
118 __tlb_flush_idte(gmap->asce); in gmap_flush_tlb()
180 static void gmap_free(struct gmap *gmap) in gmap_free() argument
185 if (!(gmap_is_shadow(gmap) && gmap->removed)) in gmap_free()
186 gmap_flush_tlb(gmap); in gmap_free()
188 list_for_each_entry_safe(page, next, &gmap->crst_list, lru) in gmap_free()
190 gmap_radix_tree_free(&gmap->guest_to_host); in gmap_free()
191 gmap_radix_tree_free(&gmap->host_to_guest); in gmap_free()
194 if (gmap_is_shadow(gmap)) { in gmap_free()
196 list_for_each_entry_safe(page, next, &gmap->pt_list, lru) in gmap_free()
198 gmap_rmap_radix_tree_free(&gmap->host_to_rmap); in gmap_free()
200 gmap_put(gmap->parent); in gmap_free()
203 kfree(gmap); in gmap_free()
212 struct gmap *gmap_get(struct gmap *gmap) in gmap_get() argument
214 atomic_inc(&gmap->ref_count); in gmap_get()
215 return gmap; in gmap_get()
225 void gmap_put(struct gmap *gmap) in gmap_put() argument
227 if (atomic_dec_return(&gmap->ref_count) == 0) in gmap_put()
228 gmap_free(gmap); in gmap_put()
236 void gmap_remove(struct gmap *gmap) in gmap_remove() argument
238 struct gmap *sg, *next; in gmap_remove()
242 if (!list_empty(&gmap->children)) { in gmap_remove()
243 spin_lock(&gmap->shadow_lock); in gmap_remove()
244 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
248 spin_unlock(&gmap->shadow_lock); in gmap_remove()
251 spin_lock(&gmap->mm->context.lock); in gmap_remove()
252 list_del_rcu(&gmap->list); in gmap_remove()
253 if (list_empty(&gmap->mm->context.gmap_list)) in gmap_remove()
255 else if (list_is_singular(&gmap->mm->context.gmap_list)) in gmap_remove()
256 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, in gmap_remove()
257 struct gmap, list)->asce; in gmap_remove()
260 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); in gmap_remove()
261 spin_unlock(&gmap->mm->context.lock); in gmap_remove()
264 gmap_put(gmap); in gmap_remove()
272 void gmap_enable(struct gmap *gmap) in gmap_enable() argument
274 S390_lowcore.gmap = (unsigned long) gmap; in gmap_enable()
282 void gmap_disable(struct gmap *gmap) in gmap_disable() argument
284 S390_lowcore.gmap = 0UL; in gmap_disable()
293 struct gmap *gmap_get_enabled(void) in gmap_get_enabled()
295 return (struct gmap *) S390_lowcore.gmap; in gmap_get_enabled()
302 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, in gmap_alloc_table() argument
314 spin_lock(&gmap->guest_table_lock); in gmap_alloc_table()
316 list_add(&page->lru, &gmap->crst_list); in gmap_alloc_table()
322 spin_unlock(&gmap->guest_table_lock); in gmap_alloc_table()
353 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) in __gmap_unlink_by_vmaddr() argument
358 BUG_ON(gmap_is_shadow(gmap)); in __gmap_unlink_by_vmaddr()
359 spin_lock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
360 entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in __gmap_unlink_by_vmaddr()
365 spin_unlock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
376 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr) in __gmap_unmap_by_gaddr() argument
380 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, in __gmap_unmap_by_gaddr()
382 return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0; in __gmap_unmap_by_gaddr()
393 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) in gmap_unmap_segment() argument
398 BUG_ON(gmap_is_shadow(gmap)); in gmap_unmap_segment()
405 down_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
407 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_unmap_segment()
408 up_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
410 gmap_flush_tlb(gmap); in gmap_unmap_segment()
424 int gmap_map_segment(struct gmap *gmap, unsigned long from, in gmap_map_segment() argument
430 BUG_ON(gmap_is_shadow(gmap)); in gmap_map_segment()
434 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) in gmap_map_segment()
438 down_write(&gmap->mm->mmap_sem); in gmap_map_segment()
441 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_map_segment()
443 if (radix_tree_insert(&gmap->guest_to_host, in gmap_map_segment()
448 up_write(&gmap->mm->mmap_sem); in gmap_map_segment()
450 gmap_flush_tlb(gmap); in gmap_map_segment()
453 gmap_unmap_segment(gmap, to, len); in gmap_map_segment()
471 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr) in __gmap_translate() argument
476 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); in __gmap_translate()
491 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr) in gmap_translate() argument
495 down_read(&gmap->mm->mmap_sem); in gmap_translate()
496 rc = __gmap_translate(gmap, gaddr); in gmap_translate()
497 up_read(&gmap->mm->mmap_sem); in gmap_translate()
511 struct gmap *gmap; in gmap_unlink() local
515 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_unlink()
516 flush = __gmap_unlink_by_vmaddr(gmap, vmaddr); in gmap_unlink()
518 gmap_flush_tlb(gmap); in gmap_unlink()
534 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) in __gmap_link() argument
545 BUG_ON(gmap_is_shadow(gmap)); in __gmap_link()
547 table = gmap->table; in __gmap_link()
548 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { in __gmap_link()
551 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, in __gmap_link()
556 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { in __gmap_link()
559 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, in __gmap_link()
564 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { in __gmap_link()
567 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, in __gmap_link()
574 mm = gmap->mm; in __gmap_link()
594 spin_lock(&gmap->guest_table_lock); in __gmap_link()
596 rc = radix_tree_insert(&gmap->host_to_guest, in __gmap_link()
602 spin_unlock(&gmap->guest_table_lock); in __gmap_link()
617 int gmap_fault(struct gmap *gmap, unsigned long gaddr, in gmap_fault() argument
624 down_read(&gmap->mm->mmap_sem); in gmap_fault()
628 vmaddr = __gmap_translate(gmap, gaddr); in gmap_fault()
633 if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags, in gmap_fault()
645 rc = __gmap_link(gmap, gaddr, vmaddr); in gmap_fault()
647 up_read(&gmap->mm->mmap_sem); in gmap_fault()
655 void __gmap_zap(struct gmap *gmap, unsigned long gaddr) in __gmap_zap() argument
662 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, in __gmap_zap()
667 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl); in __gmap_zap()
669 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0); in __gmap_zap()
675 void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) in gmap_discard() argument
680 down_read(&gmap->mm->mmap_sem); in gmap_discard()
685 radix_tree_lookup(&gmap->guest_to_host, in gmap_discard()
691 vma = find_vma(gmap->mm, vmaddr); in gmap_discard()
697 up_read(&gmap->mm->mmap_sem); in gmap_discard()
735 static void gmap_call_notifier(struct gmap *gmap, unsigned long start, in gmap_call_notifier() argument
741 nb->notifier_call(gmap, start, end); in gmap_call_notifier()
762 static inline unsigned long *gmap_table_walk(struct gmap *gmap, in gmap_table_walk() argument
767 if ((gmap->asce & _ASCE_TYPE_MASK) + 4 < (level * 4)) in gmap_table_walk()
769 if (gmap_is_shadow(gmap) && gmap->removed) in gmap_table_walk()
771 if (gaddr & (-1UL << (31 + ((gmap->asce & _ASCE_TYPE_MASK) >> 2)*11))) in gmap_table_walk()
773 table = gmap->table; in gmap_table_walk()
774 switch (gmap->asce & _ASCE_TYPE_MASK) { in gmap_table_walk()
822 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_walk() argument
827 if (gmap_is_shadow(gmap)) in gmap_pte_op_walk()
828 spin_lock(&gmap->guest_table_lock); in gmap_pte_op_walk()
830 table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */ in gmap_pte_op_walk()
832 if (gmap_is_shadow(gmap)) in gmap_pte_op_walk()
833 spin_unlock(&gmap->guest_table_lock); in gmap_pte_op_walk()
836 if (gmap_is_shadow(gmap)) { in gmap_pte_op_walk()
837 *ptl = &gmap->guest_table_lock; in gmap_pte_op_walk()
840 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); in gmap_pte_op_walk()
854 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_fixup() argument
857 struct mm_struct *mm = gmap->mm; in gmap_pte_op_fixup()
861 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_fixup()
869 return __gmap_link(gmap, gaddr, vmaddr); in gmap_pte_op_fixup()
896 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, in gmap_protect_range() argument
906 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); in gmap_protect_range()
908 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, bits); in gmap_protect_range()
912 vmaddr = __gmap_translate(gmap, gaddr); in gmap_protect_range()
915 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot); in gmap_protect_range()
940 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, in gmap_mprotect_notify() argument
945 if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap)) in gmap_mprotect_notify()
949 down_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
950 rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT); in gmap_mprotect_notify()
951 up_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
968 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) in gmap_read_table() argument
977 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); in gmap_read_table()
992 vmaddr = __gmap_translate(gmap, gaddr); in gmap_read_table()
997 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ); in gmap_read_table()
1013 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap()
1042 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap()
1045 struct gmap *parent; in gmap_protect_rmap()
1123 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page()
1143 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt()
1160 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt()
1189 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt()
1218 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt()
1247 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t()
1276 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t()
1305 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t()
1334 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t()
1363 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t()
1393 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow()
1430 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, in gmap_find_shadow()
1433 struct gmap *sg; in gmap_find_shadow()
1459 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid()
1482 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, in gmap_shadow()
1485 struct gmap *sg, *new; in gmap_shadow()
1573 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t()
1657 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t()
1740 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt()
1824 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup()
1864 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt()
1943 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page()
1945 struct gmap *parent; in gmap_shadow_page()
2011 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify()
2078 struct gmap *gmap, *sg, *next; in ptep_notify() local
2083 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in ptep_notify()
2084 spin_lock(&gmap->guest_table_lock); in ptep_notify()
2085 table = radix_tree_lookup(&gmap->host_to_guest, in ptep_notify()
2089 spin_unlock(&gmap->guest_table_lock); in ptep_notify()
2093 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { in ptep_notify()
2094 spin_lock(&gmap->shadow_lock); in ptep_notify()
2096 &gmap->children, list) in ptep_notify()
2098 spin_unlock(&gmap->shadow_lock); in ptep_notify()
2101 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); in ptep_notify()