121 	vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);  in add_sgi_source()
126 	return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);  in queue_sgi()
129 int kvm_vgic_map_resources(struct kvm *kvm)  in kvm_vgic_map_resources()  argument
131 	return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);  in kvm_vgic_map_resources()
276 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_edge()
285 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_enabled()
292 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_queued()
299 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_is_active()
306 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_set_queued()
313 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_clear_queued()
320 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_set_active()
327 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_irq_clear_active()
334 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_get_level()
341 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_set_level()
348 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_level()
355 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_soft_pend()
362 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_soft_pend()
374 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_is_pending()
381 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_set_pending()
388 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_dist_irq_clear_pending()
485 bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,  in vgic_handle_enable_reg()  argument
490 	struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);  in vgic_handle_enable_reg()
492 	reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);  in vgic_handle_enable_reg()
500 		vgic_update_state(kvm);  in vgic_handle_enable_reg()
507 bool vgic_handle_set_pending_reg(struct kvm *kvm,  in vgic_handle_set_pending_reg()  argument
514 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_set_pending_reg()
537 		vgic_update_state(kvm);  in vgic_handle_set_pending_reg()
544 bool vgic_handle_clear_pending_reg(struct kvm *kvm,  in vgic_handle_clear_pending_reg()  argument
551 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_clear_pending_reg()
574 		vgic_update_state(kvm);  in vgic_handle_clear_pending_reg()
580 bool vgic_handle_set_active_reg(struct kvm *kvm,  in vgic_handle_set_active_reg()  argument
585 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_set_active_reg()
592 		vgic_update_state(kvm);  in vgic_handle_set_active_reg()
599 bool vgic_handle_clear_active_reg(struct kvm *kvm,  in vgic_handle_clear_active_reg()  argument
604 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_handle_clear_active_reg()
611 		vgic_update_state(kvm);  in vgic_handle_clear_active_reg()
730 		vgic_update_state(vcpu->kvm);  in vgic_unqueue_irqs()
821 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_handle_mmio_access()
854 		vgic_kick_vcpus(vcpu->kvm);  in vgic_handle_mmio_access()
895 int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,  in vgic_register_kvm_io_dev()  argument
904 		vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);  in vgic_register_kvm_io_dev()
913 	mutex_lock(&kvm->slots_lock);  in vgic_register_kvm_io_dev()
915 	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,  in vgic_register_kvm_io_dev()
917 	mutex_unlock(&kvm->slots_lock);  in vgic_register_kvm_io_dev()
933 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in compute_active_for_cpu()
963 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in compute_pending_for_cpu()
1000 void vgic_update_state(struct kvm *kvm)  in vgic_update_state()  argument
1002 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_update_state()
1006 	kvm_for_each_vcpu(c, vcpu, kvm) {  in vgic_update_state()
1094 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in dist_active_irq()
1143 		vgic_update_state(vcpu->kvm);  in vgic_queue_irq_to_lr()
1181 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_queue_irq()
1245 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in __kvm_vgic_flush_hwstate()
1355 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_process_maintenance()
1356 	struct kvm *kvm = vcpu->kvm;  in vgic_process_maintenance()  local
1382 			kvm_notify_acked_irq(kvm, 0,  in vgic_process_maintenance()
1412 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in vgic_sync_hwirq()
1430 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in __kvm_vgic_sync_hwstate()
1456 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_flush_hwstate()
1458 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_flush_hwstate()
1468 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_sync_hwstate()
1476 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_vcpu_pending_irq()
1478 	if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vgic_vcpu_pending_irq()
1484 void vgic_kick_vcpus(struct kvm *kvm)  in vgic_kick_vcpus()  argument
1493 	kvm_for_each_vcpu(c, vcpu, kvm) {  in vgic_kick_vcpus()
1517 static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,  in vgic_update_irq_pending()  argument
1521 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_update_irq_pending()
1529 	if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))  in vgic_update_irq_pending()
1534 	vcpu = kvm_get_vcpu(kvm, cpuid);  in vgic_update_irq_pending()
1550 		vcpu = kvm_get_vcpu(kvm, cpuid);  in vgic_update_irq_pending()
1600 		kvm_vcpu_kick(kvm_get_vcpu(kvm, cpuid));  in vgic_update_irq_pending()
1606 static int vgic_lazy_init(struct kvm *kvm)  in vgic_lazy_init()  argument
1610 	if (unlikely(!vgic_initialized(kvm))) {  in vgic_lazy_init()
1617 		if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)  in vgic_lazy_init()
1620 		mutex_lock(&kvm->lock);  in vgic_lazy_init()
1621 		ret = vgic_init(kvm);  in vgic_lazy_init()
1622 		mutex_unlock(&kvm->lock);  in vgic_lazy_init()
1643 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,  in kvm_vgic_inject_irq()  argument
1649 	ret = vgic_lazy_init(kvm);  in kvm_vgic_inject_irq()
1653 	map = vgic_irq_map_search(kvm_get_vcpu(kvm, cpuid), irq_num);  in kvm_vgic_inject_irq()
1657 	return vgic_update_irq_pending(kvm, cpuid, NULL, irq_num, level);  in kvm_vgic_inject_irq()
1674 int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,  in kvm_vgic_inject_mapped_irq()  argument
1679 	ret = vgic_lazy_init(kvm);  in kvm_vgic_inject_mapped_irq()
1683 	return vgic_update_irq_pending(kvm, cpuid, map, map->virt_irq, level);  in kvm_vgic_inject_mapped_irq()
1703 		return &vcpu->kvm->arch.vgic.irq_phys_map_list;  in vgic_get_irq_phys_map_list()
1722 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_map_phys_irq()
1816 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;  in kvm_vgic_unmap_phys_irq()
1840 static void vgic_destroy_irq_phys_map(struct kvm *kvm, struct list_head *root)  in vgic_destroy_irq_phys_map()  argument
1842 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_destroy_irq_phys_map()
1862 	vgic_destroy_irq_phys_map(vcpu->kvm, &vgic_cpu->irq_phys_map_list);  in kvm_vgic_vcpu_destroy()
1916 void kvm_vgic_destroy(struct kvm *kvm)  in kvm_vgic_destroy()  argument
1918 	struct vgic_dist *dist = &kvm->arch.vgic;  in kvm_vgic_destroy()
1922 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_vgic_destroy()
1942 	vgic_destroy_irq_phys_map(kvm, &dist->irq_phys_map_list);  in kvm_vgic_destroy()
1955 int vgic_init(struct kvm *kvm)  in vgic_init()  argument
1957 	struct vgic_dist *dist = &kvm->arch.vgic;  in vgic_init()
1962 	if (vgic_initialized(kvm))  in vgic_init()
1965 	nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);  in vgic_init()
2014 	ret = kvm->arch.vgic.vm_ops.init_model(kvm);  in vgic_init()
2018 	kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {  in vgic_init()
2050 		kvm_vgic_destroy(kvm);  in vgic_init()
2055 static int init_vgic_model(struct kvm *kvm, int type)  in init_vgic_model()  argument
2059 		vgic_v2_init_emulation(kvm);  in init_vgic_model()
2063 		vgic_v3_init_emulation(kvm);  in init_vgic_model()
2070 	if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)  in init_vgic_model()
2081 void kvm_vgic_early_init(struct kvm *kvm)  in kvm_vgic_early_init()  argument
2083 	spin_lock_init(&kvm->arch.vgic.lock);  in kvm_vgic_early_init()
2084 	spin_lock_init(&kvm->arch.vgic.irq_phys_map_lock);  in kvm_vgic_early_init()
2085 	INIT_LIST_HEAD(&kvm->arch.vgic.irq_phys_map_list);  in kvm_vgic_early_init()
2088 int kvm_vgic_create(struct kvm *kvm, u32 type)  in kvm_vgic_create()  argument
2093 	mutex_lock(&kvm->lock);  in kvm_vgic_create()
2095 	if (irqchip_in_kernel(kvm)) {  in kvm_vgic_create()
2117 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_vgic_create()
2123 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_vgic_create()
2129 	ret = init_vgic_model(kvm, type);  in kvm_vgic_create()
2133 	kvm->arch.vgic.in_kernel = true;  in kvm_vgic_create()
2134 	kvm->arch.vgic.vgic_model = type;  in kvm_vgic_create()
2135 	kvm->arch.vgic.vctrl_base = vgic->vctrl_base;  in kvm_vgic_create()
2136 	kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2137 	kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2138 	kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;  in kvm_vgic_create()
2142 		vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);  in kvm_vgic_create()
2147 	mutex_unlock(&kvm->lock);  in kvm_vgic_create()
2151 static int vgic_ioaddr_overlap(struct kvm *kvm)  in vgic_ioaddr_overlap()  argument
2153 	phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;  in vgic_ioaddr_overlap()
2154 	phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;  in vgic_ioaddr_overlap()
2164 static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,  in vgic_ioaddr_assign()  argument
2181 	ret = vgic_ioaddr_overlap(kvm);  in vgic_ioaddr_assign()
2201 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)  in kvm_vgic_addr()  argument
2204 	struct vgic_dist *vgic = &kvm->arch.vgic;  in kvm_vgic_addr()
2209 	mutex_lock(&kvm->lock);  in kvm_vgic_addr()
2251 			r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,  in kvm_vgic_addr()
2258 	mutex_unlock(&kvm->lock);  in kvm_vgic_addr()
2275 		r = kvm_vgic_addr(dev->kvm, type, &addr, true);  in vgic_set_common_attr()
2297 		mutex_lock(&dev->kvm->lock);  in vgic_set_common_attr()
2299 		if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)  in vgic_set_common_attr()
2302 			dev->kvm->arch.vgic.nr_irqs = val;  in vgic_set_common_attr()
2304 		mutex_unlock(&dev->kvm->lock);  in vgic_set_common_attr()
2311 			r = vgic_init(dev->kvm);  in vgic_set_common_attr()
2331 		r = kvm_vgic_addr(dev->kvm, type, &addr, false);  in vgic_get_common_attr()
2342 		r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);  in vgic_get_common_attr()
2435 int kvm_irq_map_gsi(struct kvm *kvm,  in kvm_irq_map_gsi()  argument
2442 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_irq_map_chip_pin()  argument
2447 int kvm_set_irq(struct kvm *kvm, int irq_source_id,  in kvm_set_irq()  argument
2454 	BUG_ON(!vgic_initialized(kvm));  in kvm_set_irq()
2456 	return kvm_vgic_inject_irq(kvm, 0, spi, level);  in kvm_set_irq()
2461 		struct kvm *kvm, int irq_source_id,  in kvm_set_msi()  argument