Lines Matching refs:kvm
158 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
162 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, in kvm_arch_mmu_notifier_invalidate_range() argument
251 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, in kvm_make_vcpus_request_mask() argument
260 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_make_vcpus_request_mask()
281 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) in kvm_make_all_cpus_request() argument
288 called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus); in kvm_make_all_cpus_request()
295 void kvm_flush_remote_tlbs(struct kvm *kvm) in kvm_flush_remote_tlbs() argument
301 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty); in kvm_flush_remote_tlbs()
314 if (!kvm_arch_flush_remote_tlb(kvm) in kvm_flush_remote_tlbs()
315 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) in kvm_flush_remote_tlbs()
316 ++kvm->stat.remote_tlb_flush; in kvm_flush_remote_tlbs()
317 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); in kvm_flush_remote_tlbs()
322 void kvm_reload_remote_mmus(struct kvm *kvm) in kvm_reload_remote_mmus() argument
324 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD); in kvm_reload_remote_mmus()
327 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) in kvm_vcpu_init() argument
334 vcpu->kvm = kvm; in kvm_vcpu_init()
381 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) in mmu_notifier_to_kvm()
383 return container_of(mn, struct kvm, mmu_notifier); in mmu_notifier_to_kvm()
390 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range() local
393 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_invalidate_range()
394 kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); in kvm_mmu_notifier_invalidate_range()
395 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_invalidate_range()
403 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_change_pte() local
406 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_change_pte()
407 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_change_pte()
408 kvm->mmu_notifier_seq++; in kvm_mmu_notifier_change_pte()
410 if (kvm_set_spte_hva(kvm, address, pte)) in kvm_mmu_notifier_change_pte()
411 kvm_flush_remote_tlbs(kvm); in kvm_mmu_notifier_change_pte()
413 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_change_pte()
414 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_change_pte()
420 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range_start() local
423 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_invalidate_range_start()
424 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_start()
430 kvm->mmu_notifier_count++; in kvm_mmu_notifier_invalidate_range_start()
431 need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end, in kvm_mmu_notifier_invalidate_range_start()
434 if (need_tlb_flush || kvm->tlbs_dirty) in kvm_mmu_notifier_invalidate_range_start()
435 kvm_flush_remote_tlbs(kvm); in kvm_mmu_notifier_invalidate_range_start()
437 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_start()
438 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_invalidate_range_start()
446 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range_end() local
448 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_end()
454 kvm->mmu_notifier_seq++; in kvm_mmu_notifier_invalidate_range_end()
461 kvm->mmu_notifier_count--; in kvm_mmu_notifier_invalidate_range_end()
462 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_invalidate_range_end()
464 BUG_ON(kvm->mmu_notifier_count < 0); in kvm_mmu_notifier_invalidate_range_end()
472 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_clear_flush_young() local
475 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_clear_flush_young()
476 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_flush_young()
478 young = kvm_age_hva(kvm, start, end); in kvm_mmu_notifier_clear_flush_young()
480 kvm_flush_remote_tlbs(kvm); in kvm_mmu_notifier_clear_flush_young()
482 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_flush_young()
483 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_clear_flush_young()
493 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_clear_young() local
496 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_clear_young()
497 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_young()
511 young = kvm_age_hva(kvm, start, end); in kvm_mmu_notifier_clear_young()
512 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_clear_young()
513 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_clear_young()
522 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_test_young() local
525 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_test_young()
526 spin_lock(&kvm->mmu_lock); in kvm_mmu_notifier_test_young()
527 young = kvm_test_age_hva(kvm, address); in kvm_mmu_notifier_test_young()
528 spin_unlock(&kvm->mmu_lock); in kvm_mmu_notifier_test_young()
529 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_test_young()
537 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_release() local
540 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_release()
541 kvm_arch_flush_shadow_all(kvm); in kvm_mmu_notifier_release()
542 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_release()
556 static int kvm_init_mmu_notifier(struct kvm *kvm) in kvm_init_mmu_notifier() argument
558 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; in kvm_init_mmu_notifier()
559 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); in kvm_init_mmu_notifier()
564 static int kvm_init_mmu_notifier(struct kvm *kvm) in kvm_init_mmu_notifier() argument
598 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, in kvm_free_memslot() argument
604 kvm_arch_free_memslot(kvm, free, dont); in kvm_free_memslot()
609 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) in kvm_free_memslots() argument
617 kvm_free_memslot(kvm, memslot, NULL); in kvm_free_memslots()
622 static void kvm_destroy_vm_debugfs(struct kvm *kvm) in kvm_destroy_vm_debugfs() argument
626 if (!kvm->debugfs_dentry) in kvm_destroy_vm_debugfs()
629 debugfs_remove_recursive(kvm->debugfs_dentry); in kvm_destroy_vm_debugfs()
631 if (kvm->debugfs_stat_data) { in kvm_destroy_vm_debugfs()
633 kfree(kvm->debugfs_stat_data[i]); in kvm_destroy_vm_debugfs()
634 kfree(kvm->debugfs_stat_data); in kvm_destroy_vm_debugfs()
638 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) in kvm_create_vm_debugfs() argument
663 kvm->debugfs_dentry = dent; in kvm_create_vm_debugfs()
664 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, in kvm_create_vm_debugfs()
665 sizeof(*kvm->debugfs_stat_data), in kvm_create_vm_debugfs()
667 if (!kvm->debugfs_stat_data) in kvm_create_vm_debugfs()
675 stat_data->kvm = kvm; in kvm_create_vm_debugfs()
678 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; in kvm_create_vm_debugfs()
679 debugfs_create_file(p->name, stat_data->mode, kvm->debugfs_dentry, in kvm_create_vm_debugfs()
689 int __weak kvm_arch_post_init_vm(struct kvm *kvm) in kvm_arch_post_init_vm() argument
698 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm) in kvm_arch_pre_destroy_vm() argument
702 static struct kvm *kvm_create_vm(unsigned long type) in kvm_create_vm()
704 struct kvm *kvm = kvm_arch_alloc_vm(); in kvm_create_vm() local
708 if (!kvm) in kvm_create_vm()
711 spin_lock_init(&kvm->mmu_lock); in kvm_create_vm()
713 kvm->mm = current->mm; in kvm_create_vm()
714 kvm_eventfd_init(kvm); in kvm_create_vm()
715 mutex_init(&kvm->lock); in kvm_create_vm()
716 mutex_init(&kvm->irq_lock); in kvm_create_vm()
717 mutex_init(&kvm->slots_lock); in kvm_create_vm()
718 INIT_LIST_HEAD(&kvm->devices); in kvm_create_vm()
722 if (init_srcu_struct(&kvm->srcu)) in kvm_create_vm()
724 if (init_srcu_struct(&kvm->irq_srcu)) in kvm_create_vm()
727 refcount_set(&kvm->users_count, 1); in kvm_create_vm()
735 rcu_assign_pointer(kvm->memslots[i], slots); in kvm_create_vm()
739 rcu_assign_pointer(kvm->buses[i], in kvm_create_vm()
741 if (!kvm->buses[i]) in kvm_create_vm()
745 r = kvm_arch_init_vm(kvm, type); in kvm_create_vm()
754 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); in kvm_create_vm()
757 r = kvm_init_mmu_notifier(kvm); in kvm_create_vm()
761 r = kvm_arch_post_init_vm(kvm); in kvm_create_vm()
766 list_add(&kvm->vm_list, &vm_list); in kvm_create_vm()
781 return kvm; in kvm_create_vm()
785 if (kvm->mmu_notifier.ops) in kvm_create_vm()
786 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); in kvm_create_vm()
791 kvm_arch_destroy_vm(kvm); in kvm_create_vm()
793 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); in kvm_create_vm()
795 kfree(kvm_get_bus(kvm, i)); in kvm_create_vm()
797 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); in kvm_create_vm()
798 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_create_vm()
800 cleanup_srcu_struct(&kvm->srcu); in kvm_create_vm()
802 kvm_arch_free_vm(kvm); in kvm_create_vm()
807 static void kvm_destroy_devices(struct kvm *kvm) in kvm_destroy_devices() argument
816 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { in kvm_destroy_devices()
822 static void kvm_destroy_vm(struct kvm *kvm) in kvm_destroy_vm() argument
825 struct mm_struct *mm = kvm->mm; in kvm_destroy_vm()
827 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); in kvm_destroy_vm()
828 kvm_destroy_vm_debugfs(kvm); in kvm_destroy_vm()
829 kvm_arch_sync_events(kvm); in kvm_destroy_vm()
831 list_del(&kvm->vm_list); in kvm_destroy_vm()
833 kvm_arch_pre_destroy_vm(kvm); in kvm_destroy_vm()
835 kvm_free_irq_routing(kvm); in kvm_destroy_vm()
837 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); in kvm_destroy_vm()
841 kvm->buses[i] = NULL; in kvm_destroy_vm()
843 kvm_coalesced_mmio_free(kvm); in kvm_destroy_vm()
845 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); in kvm_destroy_vm()
847 kvm_arch_flush_shadow_all(kvm); in kvm_destroy_vm()
849 kvm_arch_destroy_vm(kvm); in kvm_destroy_vm()
850 kvm_destroy_devices(kvm); in kvm_destroy_vm()
852 kvm_free_memslots(kvm, __kvm_memslots(kvm, i)); in kvm_destroy_vm()
853 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_destroy_vm()
854 cleanup_srcu_struct(&kvm->srcu); in kvm_destroy_vm()
855 kvm_arch_free_vm(kvm); in kvm_destroy_vm()
862 void kvm_get_kvm(struct kvm *kvm) in kvm_get_kvm() argument
864 refcount_inc(&kvm->users_count); in kvm_get_kvm()
868 void kvm_put_kvm(struct kvm *kvm) in kvm_put_kvm() argument
870 if (refcount_dec_and_test(&kvm->users_count)) in kvm_put_kvm()
871 kvm_destroy_vm(kvm); in kvm_put_kvm()
878 struct kvm *kvm = filp->private_data; in kvm_vm_release() local
880 kvm_irqfd_release(kvm); in kvm_vm_release()
882 kvm_put_kvm(kvm); in kvm_vm_release()
975 static struct kvm_memslots *install_new_memslots(struct kvm *kvm, in install_new_memslots() argument
978 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id); in install_new_memslots()
984 rcu_assign_pointer(kvm->memslots[as_id], slots); in install_new_memslots()
985 synchronize_srcu_expedited(&kvm->srcu); in install_new_memslots()
1004 kvm_arch_memslots_updated(kvm, gen); in install_new_memslots()
1019 int __kvm_set_memory_region(struct kvm *kvm, in __kvm_set_memory_region() argument
1056 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); in __kvm_set_memory_region()
1100 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { in __kvm_set_memory_region()
1117 if (kvm_arch_create_memslot(kvm, &new, npages)) in __kvm_set_memory_region()
1130 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); in __kvm_set_memory_region()
1136 old_memslots = install_new_memslots(kvm, as_id, slots); in __kvm_set_memory_region()
1145 kvm_arch_flush_shadow_memslot(kvm, slot); in __kvm_set_memory_region()
1155 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); in __kvm_set_memory_region()
1166 old_memslots = install_new_memslots(kvm, as_id, slots); in __kvm_set_memory_region()
1168 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); in __kvm_set_memory_region()
1170 kvm_free_memslot(kvm, &old, &new); in __kvm_set_memory_region()
1177 kvm_free_memslot(kvm, &new, &old); in __kvm_set_memory_region()
1183 int kvm_set_memory_region(struct kvm *kvm, in kvm_set_memory_region() argument
1188 mutex_lock(&kvm->slots_lock); in kvm_set_memory_region()
1189 r = __kvm_set_memory_region(kvm, mem); in kvm_set_memory_region()
1190 mutex_unlock(&kvm->slots_lock); in kvm_set_memory_region()
1195 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, in kvm_vm_ioctl_set_memory_region() argument
1201 return kvm_set_memory_region(kvm, mem); in kvm_vm_ioctl_set_memory_region()
1204 int kvm_get_dirty_log(struct kvm *kvm, in kvm_get_dirty_log() argument
1218 slots = __kvm_memslots(kvm, as_id); in kvm_get_dirty_log()
1260 int kvm_get_dirty_log_protect(struct kvm *kvm, in kvm_get_dirty_log_protect() argument
1275 slots = __kvm_memslots(kvm, as_id); in kvm_get_dirty_log_protect()
1284 if (kvm->manual_dirty_log_protect) { in kvm_get_dirty_log_protect()
1298 spin_lock(&kvm->mmu_lock); in kvm_get_dirty_log_protect()
1311 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, in kvm_get_dirty_log_protect()
1314 spin_unlock(&kvm->mmu_lock); in kvm_get_dirty_log_protect()
1330 int kvm_clear_dirty_log_protect(struct kvm *kvm, in kvm_clear_dirty_log_protect() argument
1349 slots = __kvm_memslots(kvm, as_id); in kvm_clear_dirty_log_protect()
1368 spin_lock(&kvm->mmu_lock); in kvm_clear_dirty_log_protect()
1387 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, in kvm_clear_dirty_log_protect()
1391 spin_unlock(&kvm->mmu_lock); in kvm_clear_dirty_log_protect()
1409 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) in gfn_to_memslot() argument
1411 return __gfn_to_memslot(kvm_memslots(kvm), gfn); in gfn_to_memslot()
1420 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) in kvm_is_visible_gfn() argument
1422 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); in kvm_is_visible_gfn()
1489 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) in gfn_to_hva() argument
1491 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); in gfn_to_hva()
1520 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) in gfn_to_hva_prot() argument
1522 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in gfn_to_hva_prot()
1787 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, in gfn_to_pfn_prot() argument
1790 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL, in gfn_to_pfn_prot()
1807 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) in gfn_to_pfn_atomic() argument
1809 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); in gfn_to_pfn_atomic()
1819 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) in gfn_to_pfn() argument
1821 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); in gfn_to_pfn()
1861 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) in gfn_to_page() argument
1865 pfn = gfn_to_pfn(kvm, gfn); in gfn_to_page()
1954 return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map, in kvm_map_gfn()
2005 __kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map, in kvm_unmap_gfn()
2105 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, in kvm_read_guest_page() argument
2108 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in kvm_read_guest_page()
2123 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) in kvm_read_guest() argument
2131 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); in kvm_read_guest()
2180 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, in kvm_read_guest_atomic() argument
2184 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in kvm_read_guest_atomic()
2218 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, in kvm_write_guest_page() argument
2221 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in kvm_write_guest_page()
2236 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, in kvm_write_guest() argument
2245 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); in kvm_write_guest()
2316 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_gfn_to_hva_cache_init() argument
2319 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_gfn_to_hva_cache_init()
2324 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_write_guest_offset_cached() argument
2328 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_write_guest_offset_cached()
2341 return kvm_write_guest(kvm, gpa, data, len); in kvm_write_guest_offset_cached()
2352 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_write_guest_cached() argument
2355 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); in kvm_write_guest_cached()
2359 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_read_guest_cached() argument
2362 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_read_guest_cached()
2374 return kvm_read_guest(kvm, ghc->gpa, data, len); in kvm_read_guest_cached()
2384 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) in kvm_clear_guest_page() argument
2388 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len); in kvm_clear_guest_page()
2392 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) in kvm_clear_guest() argument
2400 ret = kvm_clear_guest_page(kvm, gfn, offset, seg); in kvm_clear_guest()
2421 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) in mark_page_dirty() argument
2425 memslot = gfn_to_memslot(kvm, gfn); in mark_page_dirty()
2506 int idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_vcpu_check_block()
2519 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_vcpu_check_block()
2715 struct kvm *kvm = me->kvm; in kvm_vcpu_on_spin() local
2717 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; in kvm_vcpu_on_spin()
2732 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_vcpu_on_spin()
2752 kvm->last_boosted_vcpu = i; in kvm_vcpu_on_spin()
2781 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); in kvm_vcpu_fault()
2805 kvm_put_kvm(vcpu->kvm); in kvm_vcpu_release()
2838 vcpu->kvm->debugfs_dentry); in kvm_create_vcpu_debugfs()
2847 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) in kvm_vm_ioctl_create_vcpu() argument
2855 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2856 if (kvm->created_vcpus == KVM_MAX_VCPUS) { in kvm_vm_ioctl_create_vcpu()
2857 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2861 kvm->created_vcpus++; in kvm_vm_ioctl_create_vcpu()
2862 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2864 vcpu = kvm_arch_vcpu_create(kvm, id); in kvm_vm_ioctl_create_vcpu()
2878 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2879 if (kvm_get_vcpu_by_id(kvm, id)) { in kvm_vm_ioctl_create_vcpu()
2884 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
2885 BUG_ON(kvm->vcpus[vcpu->vcpu_idx]); in kvm_vm_ioctl_create_vcpu()
2888 kvm_get_kvm(kvm); in kvm_vm_ioctl_create_vcpu()
2891 kvm_put_kvm(kvm); in kvm_vm_ioctl_create_vcpu()
2895 kvm->vcpus[vcpu->vcpu_idx] = vcpu; in kvm_vm_ioctl_create_vcpu()
2902 atomic_inc(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
2904 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2909 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2914 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2915 kvm->created_vcpus--; in kvm_vm_ioctl_create_vcpu()
2916 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
2940 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_bugged) in kvm_vcpu_ioctl()
3147 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_bugged) in kvm_vcpu_compat_ioctl()
3212 if (dev->kvm->mm != current->mm || dev->kvm->vm_bugged) in kvm_device_ioctl()
3233 struct kvm *kvm = dev->kvm; in kvm_device_release() local
3236 mutex_lock(&kvm->lock); in kvm_device_release()
3239 mutex_unlock(&kvm->lock); in kvm_device_release()
3242 kvm_put_kvm(kvm); in kvm_device_release()
3286 static int kvm_ioctl_create_device(struct kvm *kvm, in kvm_ioctl_create_device() argument
3311 dev->kvm = kvm; in kvm_ioctl_create_device()
3313 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
3316 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3320 list_add(&dev->vm_node, &kvm->devices); in kvm_ioctl_create_device()
3321 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3326 kvm_get_kvm(kvm); in kvm_ioctl_create_device()
3329 kvm_put_kvm(kvm); in kvm_ioctl_create_device()
3330 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
3334 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
3344 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) in kvm_vm_ioctl_check_extension_generic() argument
3384 return kvm_vm_ioctl_check_extension(kvm, arg); in kvm_vm_ioctl_check_extension_generic()
3387 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, in kvm_vm_ioctl_enable_cap() argument
3393 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, in kvm_vm_ioctl_enable_cap_generic() argument
3401 kvm->manual_dirty_log_protect = cap->args[0]; in kvm_vm_ioctl_enable_cap_generic()
3405 return kvm_vm_ioctl_enable_cap(kvm, cap); in kvm_vm_ioctl_enable_cap_generic()
3412 struct kvm *kvm = filp->private_data; in kvm_vm_ioctl() local
3416 if (kvm->mm != current->mm || kvm->vm_bugged) in kvm_vm_ioctl()
3420 r = kvm_vm_ioctl_create_vcpu(kvm, arg); in kvm_vm_ioctl()
3428 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap); in kvm_vm_ioctl()
3439 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); in kvm_vm_ioctl()
3448 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); in kvm_vm_ioctl()
3458 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); in kvm_vm_ioctl()
3469 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); in kvm_vm_ioctl()
3478 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); in kvm_vm_ioctl()
3488 r = kvm_irqfd(kvm, &data); in kvm_vm_ioctl()
3497 r = kvm_ioeventfd(kvm, &data); in kvm_vm_ioctl()
3507 r = kvm_send_userspace_msi(kvm, &msi); in kvm_vm_ioctl()
3520 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, in kvm_vm_ioctl()
3545 if (!kvm_arch_can_set_irq_routing(kvm)) in kvm_vm_ioctl()
3563 r = kvm_set_irq_routing(kvm, entries, routing.nr, in kvm_vm_ioctl()
3577 r = kvm_ioctl_create_device(kvm, &cd); in kvm_vm_ioctl()
3589 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); in kvm_vm_ioctl()
3621 struct kvm *kvm = filp->private_data; in kvm_vm_compat_ioctl() local
3624 if (kvm->mm != current->mm || kvm->vm_bugged) in kvm_vm_compat_ioctl()
3641 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); in kvm_vm_compat_ioctl()
3657 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); in kvm_vm_compat_ioctl()
3677 struct kvm *kvm; in kvm_dev_ioctl_create_vm() local
3680 kvm = kvm_create_vm(type); in kvm_dev_ioctl_create_vm()
3681 if (IS_ERR(kvm)) in kvm_dev_ioctl_create_vm()
3682 return PTR_ERR(kvm); in kvm_dev_ioctl_create_vm()
3684 r = kvm_coalesced_mmio_init(kvm); in kvm_dev_ioctl_create_vm()
3692 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); in kvm_dev_ioctl_create_vm()
3705 if (kvm_create_vm_debugfs(kvm, r) < 0) { in kvm_dev_ioctl_create_vm()
3710 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); in kvm_dev_ioctl_create_vm()
3716 kvm_put_kvm(kvm); in kvm_dev_ioctl_create_vm()
3976 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write()
3996 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write_cookie()
4047 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_read()
4055 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, in kvm_io_bus_register_dev() argument
4062 bus = kvm_get_bus(kvm, bus_idx); in kvm_io_bus_register_dev()
4090 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_register_dev()
4091 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_register_dev()
4098 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, in kvm_io_bus_unregister_dev() argument
4104 bus = kvm_get_bus(kvm, bus_idx); in kvm_io_bus_unregister_dev()
4132 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_unregister_dev()
4133 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_unregister_dev()
4138 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, in kvm_io_bus_get_dev() argument
4145 srcu_idx = srcu_read_lock(&kvm->srcu); in kvm_io_bus_get_dev()
4147 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); in kvm_io_bus_get_dev()
4158 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvm_io_bus_get_dev()
4176 if (!refcount_inc_not_zero(&stat_data->kvm->users_count)) in kvm_debugfs_open()
4182 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_open()
4195 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_release()
4204 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset); in vm_stat_get_per_vm()
4216 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0; in vm_stat_clear_per_vm()
4245 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) in vcpu_stat_get_per_vm()
4260 kvm_for_each_vcpu(i, vcpu, stat_data->kvm) in vcpu_stat_clear_per_vm()
4290 struct kvm *kvm; in vm_stat_get() local
4296 list_for_each_entry(kvm, &vm_list, vm_list) { in vm_stat_get()
4297 stat_tmp.kvm = kvm; in vm_stat_get()
4308 struct kvm *kvm; in vm_stat_clear() local
4315 list_for_each_entry(kvm, &vm_list, vm_list) { in vm_stat_clear()
4316 stat_tmp.kvm = kvm; in vm_stat_clear()
4329 struct kvm *kvm; in vcpu_stat_get() local
4335 list_for_each_entry(kvm, &vm_list, vm_list) { in vcpu_stat_get()
4336 stat_tmp.kvm = kvm; in vcpu_stat_get()
4347 struct kvm *kvm; in vcpu_stat_clear() local
4354 list_for_each_entry(kvm, &vm_list, vm_list) { in vcpu_stat_clear()
4355 stat_tmp.kvm = kvm; in vcpu_stat_clear()
4371 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) in kvm_uevent_notify_change() argument
4376 if (!kvm_dev.this_device || !kvm) in kvm_uevent_notify_change()
4399 kvm->userspace_pid = task_pid_nr(current); in kvm_uevent_notify_change()
4403 add_uevent_var(env, "PID=%d", kvm->userspace_pid); in kvm_uevent_notify_change()
4405 if (kvm->debugfs_dentry) { in kvm_uevent_notify_change()
4409 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); in kvm_uevent_notify_change()
4615 struct kvm *kvm; member
4630 struct kvm *kvm = init_context->kvm; in kvm_vm_worker_thread() local
4662 err = thread_fn(kvm, data); in kvm_vm_worker_thread()
4667 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, in kvm_vm_create_worker_thread() argument
4675 init_context.kvm = kvm; in kvm_vm_create_worker_thread()