695 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)  in kvm_destroy_dirty_bitmap()  argument
697 	if (!memslot->dirty_bitmap)  in kvm_destroy_dirty_bitmap()
700 	kvfree(memslot->dirty_bitmap);  in kvm_destroy_dirty_bitmap()
701 	memslot->dirty_bitmap = NULL;  in kvm_destroy_dirty_bitmap()
716 	struct kvm_memory_slot *memslot;  in kvm_free_memslots()  local
721 	kvm_for_each_memslot(memslot, slots)  in kvm_free_memslots()
722 		kvm_free_memslot(kvm, memslot);  in kvm_free_memslots()
1009 static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)  in kvm_alloc_dirty_bitmap()  argument
1011 	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);  in kvm_alloc_dirty_bitmap()
1013 	memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);  in kvm_alloc_dirty_bitmap()
1014 	if (!memslot->dirty_bitmap)  in kvm_alloc_dirty_bitmap()
1025 				      struct kvm_memory_slot *memslot)  in kvm_memslot_delete()  argument
1030 	if (WARN_ON(slots->id_to_index[memslot->id] == -1))  in kvm_memslot_delete()
1038 	for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) {  in kvm_memslot_delete()
1042 	mslots[i] = *memslot;  in kvm_memslot_delete()
1043 	slots->id_to_index[memslot->id] = -1;  in kvm_memslot_delete()
1063 					    struct kvm_memory_slot *memslot)  in kvm_memslot_move_backward()  argument
1068 	if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) ||  in kvm_memslot_move_backward()
1077 	for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) {  in kvm_memslot_move_backward()
1078 		if (memslot->base_gfn > mslots[i + 1].base_gfn)  in kvm_memslot_move_backward()
1081 		WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn);  in kvm_memslot_move_backward()
1098 					   struct kvm_memory_slot *memslot,  in kvm_memslot_move_forward()  argument
1105 		if (memslot->base_gfn < mslots[i - 1].base_gfn)  in kvm_memslot_move_forward()
1108 		WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn);  in kvm_memslot_move_forward()
1159 			    struct kvm_memory_slot *memslot,  in update_memslots()  argument
1165 		kvm_memslot_delete(slots, memslot);  in update_memslots()
1170 			i = kvm_memslot_move_backward(slots, memslot);  in update_memslots()
1171 		i = kvm_memslot_move_forward(slots, memslot, i);  in update_memslots()
1177 		slots->memslots[i] = *memslot;  in update_memslots()
1178 		slots->id_to_index[memslot->id] = i;  in update_memslots()
1503 		      int *is_dirty, struct kvm_memory_slot **memslot)  in kvm_get_dirty_log()  argument
1510 	*memslot = NULL;  in kvm_get_dirty_log()
1519 	*memslot = id_to_memslot(slots, id);  in kvm_get_dirty_log()
1520 	if (!(*memslot) || !(*memslot)->dirty_bitmap)  in kvm_get_dirty_log()
1523 	kvm_arch_sync_dirty_log(kvm, *memslot);  in kvm_get_dirty_log()
1525 	n = kvm_dirty_bitmap_bytes(*memslot);  in kvm_get_dirty_log()
1528 		any = (*memslot)->dirty_bitmap[i];  in kvm_get_dirty_log()
1530 	if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))  in kvm_get_dirty_log()
1564 	struct kvm_memory_slot *memslot;  in kvm_get_dirty_log_protect()  local
1577 	memslot = id_to_memslot(slots, id);  in kvm_get_dirty_log_protect()
1578 	if (!memslot || !memslot->dirty_bitmap)  in kvm_get_dirty_log_protect()
1581 	dirty_bitmap = memslot->dirty_bitmap;  in kvm_get_dirty_log_protect()
1583 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_get_dirty_log_protect()
1585 	n = kvm_dirty_bitmap_bytes(memslot);  in kvm_get_dirty_log_protect()
1598 		dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);  in kvm_get_dirty_log_protect()
1614 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_get_dirty_log_protect()
1621 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_get_dirty_log_protect()
1671 	struct kvm_memory_slot *memslot;  in kvm_clear_dirty_log_protect()  local
1688 	memslot = id_to_memslot(slots, id);  in kvm_clear_dirty_log_protect()
1689 	if (!memslot || !memslot->dirty_bitmap)  in kvm_clear_dirty_log_protect()
1692 	dirty_bitmap = memslot->dirty_bitmap;  in kvm_clear_dirty_log_protect()
1696 	if (log->first_page > memslot->npages ||  in kvm_clear_dirty_log_protect()
1697 	    log->num_pages > memslot->npages - log->first_page ||  in kvm_clear_dirty_log_protect()
1698 	    (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))  in kvm_clear_dirty_log_protect()
1701 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_clear_dirty_log_protect()
1704 	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);  in kvm_clear_dirty_log_protect()
1727 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_clear_dirty_log_protect()
1734 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_clear_dirty_log_protect()
1766 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);  in kvm_is_visible_gfn()  local
1768 	return kvm_is_visible_memslot(memslot);  in kvm_is_visible_gfn()
1774 	struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_is_visible_gfn()  local
1776 	return kvm_is_visible_memslot(memslot);  in kvm_vcpu_is_visible_gfn()
2306 static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot,  in __kvm_unmap_gfn()  argument
2331 		mark_page_dirty_in_slot(memslot, map->gfn);  in __kvm_unmap_gfn()
2540 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,  in __kvm_write_guest_page()  argument
2546 	addr = gfn_to_hva_memslot(memslot, gfn);  in __kvm_write_guest_page()
2552 	mark_page_dirty_in_slot(memslot, gfn);  in __kvm_write_guest_page()
2639 		ghc->memslot = __gfn_to_memslot(slots, start_gfn);  in __kvm_gfn_to_hva_cache_init()
2640 		ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,  in __kvm_gfn_to_hva_cache_init()
2650 		ghc->memslot = NULL;  in __kvm_gfn_to_hva_cache_init()
2684 	if (unlikely(!ghc->memslot))  in kvm_write_guest_offset_cached()
2690 	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);  in kvm_write_guest_offset_cached()
2722 	if (unlikely(!ghc->memslot))  in kvm_read_guest_offset_cached()
2767 void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn)  in mark_page_dirty_in_slot()  argument
2769 	if (memslot && memslot->dirty_bitmap) {  in mark_page_dirty_in_slot()
2770 		unsigned long rel_gfn = gfn - memslot->base_gfn;  in mark_page_dirty_in_slot()
2772 		set_bit_le(rel_gfn, memslot->dirty_bitmap);  in mark_page_dirty_in_slot()
2779 	struct kvm_memory_slot *memslot;  in mark_page_dirty()  local
2781 	memslot = gfn_to_memslot(kvm, gfn);  in mark_page_dirty()
2782 	mark_page_dirty_in_slot(memslot, gfn);  in mark_page_dirty()
2788 	struct kvm_memory_slot *memslot;  in kvm_vcpu_mark_page_dirty()  local
2790 	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_mark_page_dirty()
2791 	mark_page_dirty_in_slot(memslot, gfn);  in kvm_vcpu_mark_page_dirty()