Lines Matching +full:reserved +full:- +full:ipi +full:- +full:vectors
18 * the COPYING file in the top-level directory.
47 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
61 #define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
80 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_pending_eoi()
82 return apic_test_vector(vector, apic->regs + APIC_ISR) || in kvm_apic_pending_eoi()
83 apic_test_vector(vector, apic->regs + APIC_IRR); in kvm_apic_pending_eoi()
123 return apic->vcpu->vcpu_id; in kvm_x2apic_id()
128 switch (map->mode) { in kvm_apic_map_get_logical_dest()
131 u32 max_apic_id = map->max_apic_id; in kvm_apic_map_get_logical_dest()
134 u8 cluster_size = min(max_apic_id - offset + 1, 16U); in kvm_apic_map_get_logical_dest()
136 offset = array_index_nospec(offset, map->max_apic_id + 1); in kvm_apic_map_get_logical_dest()
137 *cluster = &map->phys_map[offset]; in kvm_apic_map_get_logical_dest()
138 *mask = dest_id & (0xffff >> (16 - cluster_size)); in kvm_apic_map_get_logical_dest()
146 *cluster = map->xapic_flat_map; in kvm_apic_map_get_logical_dest()
150 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; in kvm_apic_map_get_logical_dest()
173 mutex_lock(&kvm->arch.apic_map_lock); in recalculate_apic_map()
177 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); in recalculate_apic_map()
185 new->max_apic_id = max_id; in recalculate_apic_map()
188 struct kvm_lapic *apic = vcpu->arch.apic; in recalculate_apic_map()
203 x2apic_id <= new->max_apic_id) in recalculate_apic_map()
204 new->phys_map[x2apic_id] = apic; in recalculate_apic_map()
206 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around, in recalculate_apic_map()
209 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) in recalculate_apic_map()
210 new->phys_map[xapic_id] = apic; in recalculate_apic_map()
218 new->mode |= KVM_APIC_MODE_X2APIC; in recalculate_apic_map()
222 new->mode |= KVM_APIC_MODE_XAPIC_FLAT; in recalculate_apic_map()
224 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER; in recalculate_apic_map()
231 cluster[ffs(mask) - 1] = apic; in recalculate_apic_map()
234 old = rcu_dereference_protected(kvm->arch.apic_map, in recalculate_apic_map()
235 lockdep_is_held(&kvm->arch.apic_map_lock)); in recalculate_apic_map()
236 rcu_assign_pointer(kvm->arch.apic_map, new); in recalculate_apic_map()
237 mutex_unlock(&kvm->arch.apic_map_lock); in recalculate_apic_map()
240 call_rcu(&old->rcu, kvm_apic_map_free); in recalculate_apic_map()
251 if (enabled != apic->sw_enabled) { in apic_set_spiv()
252 apic->sw_enabled = enabled; in apic_set_spiv()
255 recalculate_apic_map(apic->vcpu->kvm); in apic_set_spiv()
259 recalculate_apic_map(apic->vcpu->kvm); in apic_set_spiv()
266 recalculate_apic_map(apic->vcpu->kvm); in kvm_apic_set_xapic_id()
272 recalculate_apic_map(apic->vcpu->kvm); in kvm_apic_set_ldr()
284 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); in kvm_apic_set_x2apic_id()
288 recalculate_apic_map(apic->vcpu->kvm); in kvm_apic_set_x2apic_id()
303 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; in apic_lvtt_oneshot()
308 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; in apic_lvtt_period()
313 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; in apic_lvtt_tscdeadline()
323 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_version()
331 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) in kvm_apic_set_version()
333 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC in kvm_apic_set_version()
334 * version first and level-triggered interrupts never get EOIed in in kvm_apic_set_version()
337 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); in kvm_apic_set_version()
338 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) && in kvm_apic_set_version()
339 !ioapic_in_kernel(vcpu->kvm)) in kvm_apic_set_version()
348 LINT_MASK, LINT_MASK, /* LVT0-1 */
357 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; in find_highest_vector()
358 vec >= 0; vec -= APIC_VECTORS_PER_REG) { in find_highest_vector()
364 return -1; in find_highest_vector()
387 max_updated_irr = -1; in __kvm_apic_update_irr()
388 *max_irr = -1; in __kvm_apic_update_irr()
406 return ((max_updated_irr != -1) && in __kvm_apic_update_irr()
413 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_irr()
415 return __kvm_apic_update_irr(pir, apic->regs, max_irr); in kvm_apic_update_irr()
421 return find_highest_vector(apic->regs + APIC_IRR); in apic_search_irr()
432 if (!apic->irr_pending) in apic_find_highest_irr()
433 return -1; in apic_find_highest_irr()
436 ASSERT(result == -1 || result >= 16); in apic_find_highest_irr()
445 vcpu = apic->vcpu; in apic_clear_irr()
447 if (unlikely(vcpu->arch.apicv_active)) { in apic_clear_irr()
449 apic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
450 kvm_x86_ops->hwapic_irr_update(vcpu, in apic_clear_irr()
453 apic->irr_pending = false; in apic_clear_irr()
454 apic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
455 if (apic_search_irr(apic) != -1) in apic_clear_irr()
456 apic->irr_pending = true; in apic_clear_irr()
464 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) in apic_set_isr()
467 vcpu = apic->vcpu; in apic_set_isr()
474 if (unlikely(vcpu->arch.apicv_active)) in apic_set_isr()
475 kvm_x86_ops->hwapic_isr_update(vcpu, vec); in apic_set_isr()
477 ++apic->isr_count; in apic_set_isr()
478 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); in apic_set_isr()
484 apic->highest_isr_cache = vec; in apic_set_isr()
494 * is always -1, with APIC virtualization enabled. in apic_find_highest_isr()
496 if (!apic->isr_count) in apic_find_highest_isr()
497 return -1; in apic_find_highest_isr()
498 if (likely(apic->highest_isr_cache != -1)) in apic_find_highest_isr()
499 return apic->highest_isr_cache; in apic_find_highest_isr()
501 result = find_highest_vector(apic->regs + APIC_ISR); in apic_find_highest_isr()
502 ASSERT(result == -1 || result >= 16); in apic_find_highest_isr()
510 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) in apic_clear_isr()
513 vcpu = apic->vcpu; in apic_clear_isr()
517 * uses the Hyper-V APIC enlightenment. In this case we may need in apic_clear_isr()
522 if (unlikely(vcpu->arch.apicv_active)) in apic_clear_isr()
523 kvm_x86_ops->hwapic_isr_update(vcpu, in apic_clear_isr()
526 --apic->isr_count; in apic_clear_isr()
527 BUG_ON(apic->isr_count < 0); in apic_clear_isr()
528 apic->highest_isr_cache = -1; in apic_clear_isr()
539 return apic_find_highest_irr(vcpu->arch.apic); in kvm_lapic_find_highest_irr()
550 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_irq()
552 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, in kvm_apic_set_irq()
553 irq->level, irq->trig_mode, dest_map); in kvm_apic_set_irq()
573 return -KVM_EINVAL; in kvm_pv_send_ipi()
575 return -KVM_EINVAL; in kvm_pv_send_ipi()
578 map = rcu_dereference(kvm->arch.apic_map); in kvm_pv_send_ipi()
581 count = -EOPNOTSUPP; in kvm_pv_send_ipi()
585 if (min > map->max_apic_id) in kvm_pv_send_ipi()
589 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { in kvm_pv_send_ipi()
590 if (map->phys_map[min + i]) { in kvm_pv_send_ipi()
591 vcpu = map->phys_map[min + i]->vcpu; in kvm_pv_send_ipi()
598 if (min > map->max_apic_id) in kvm_pv_send_ipi()
602 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { in kvm_pv_send_ipi()
603 if (map->phys_map[min + i]) { in kvm_pv_send_ipi()
604 vcpu = map->phys_map[min + i]->vcpu; in kvm_pv_send_ipi()
617 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, in pv_eoi_put_user()
624 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, in pv_eoi_get_user()
630 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; in pv_eoi_enabled()
638 (unsigned long long)vcpu->arch.pv_eoi.msr_val); in pv_eoi_get_pending()
648 (unsigned long long)vcpu->arch.pv_eoi.msr_val); in pv_eoi_set_pending()
651 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); in pv_eoi_set_pending()
658 (unsigned long long)vcpu->arch.pv_eoi.msr_val); in pv_eoi_clr_pending()
661 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); in pv_eoi_clr_pending()
667 if (apic->vcpu->arch.apicv_active) in apic_has_interrupt_for_ppr()
668 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); in apic_has_interrupt_for_ppr()
671 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) in apic_has_interrupt_for_ppr()
672 return -1; in apic_has_interrupt_for_ppr()
684 isrv = (isr != -1) ? isr : 0; in __apic_update_ppr()
706 apic_has_interrupt_for_ppr(apic, ppr) != -1) in apic_update_ppr()
707 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_update_ppr()
712 apic_update_ppr(vcpu->arch.apic); in kvm_apic_update_ppr()
771 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR)); in kvm_apic_match_logical_addr()
778 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
782 * - in-kernel IOAPIC messages have to be delivered directly to
785 * rewrites the destination of non-IPI messages from APIC_BROADCAST
789 * important when userspace wants to use x2APIC-format MSIs, because
790 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
795 bool ipi = source != NULL; in kvm_apic_mda() local
797 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && in kvm_apic_mda()
798 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) in kvm_apic_mda()
807 struct kvm_lapic *target = vcpu->arch.apic; in kvm_apic_match_dest()
839 int i, idx = -1; in kvm_vector_to_index()
853 if (!kvm->arch.disabled_lapic_found) { in kvm_apic_disabled_lapic_found()
854 kvm->arch.disabled_lapic_found = true; in kvm_apic_disabled_lapic_found()
863 if (kvm->arch.x2apic_broadcast_quirk_disabled) { in kvm_apic_is_broadcast_dest()
864 if ((irq->dest_id == APIC_BROADCAST && in kvm_apic_is_broadcast_dest()
865 map->mode != KVM_APIC_MODE_X2APIC)) in kvm_apic_is_broadcast_dest()
867 if (irq->dest_id == X2APIC_BROADCAST) in kvm_apic_is_broadcast_dest()
871 if (irq->dest_id == (x2apic_ipi ? in kvm_apic_is_broadcast_dest()
893 if (irq->shorthand == APIC_DEST_SELF && src) { in kvm_apic_map_get_dest_lapic()
897 } else if (irq->shorthand) in kvm_apic_map_get_dest_lapic()
903 if (irq->dest_mode == APIC_DEST_PHYSICAL) { in kvm_apic_map_get_dest_lapic()
904 if (irq->dest_id > map->max_apic_id) { in kvm_apic_map_get_dest_lapic()
907 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); in kvm_apic_map_get_dest_lapic()
908 *dst = &map->phys_map[dest_id]; in kvm_apic_map_get_dest_lapic()
915 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, in kvm_apic_map_get_dest_lapic()
923 lowest = -1; in kvm_apic_map_get_dest_lapic()
929 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, in kvm_apic_map_get_dest_lapic()
930 (*dst)[lowest]->vcpu) < 0) in kvm_apic_map_get_dest_lapic()
937 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), in kvm_apic_map_get_dest_lapic()
961 *r = -1; in kvm_irq_delivery_to_apic_fast()
963 if (irq->shorthand == APIC_DEST_SELF) { in kvm_irq_delivery_to_apic_fast()
964 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); in kvm_irq_delivery_to_apic_fast()
969 map = rcu_dereference(kvm->arch.apic_map); in kvm_irq_delivery_to_apic_fast()
978 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); in kvm_irq_delivery_to_apic_fast()
988 * - For single-destination interrupts, handle it in posted mode
989 * - Else if vector hashing is enabled and it is a lowest-priority
992 * 1. For lowest-priority interrupts, store all the possible
995 * the right destination vCPU in the array for the lowest-priority
997 * - Otherwise, use remapped mode to inject the interrupt.
1007 if (irq->shorthand) in kvm_intr_is_single_vcpu_fast()
1011 map = rcu_dereference(kvm->arch.apic_map); in kvm_intr_is_single_vcpu_fast()
1018 *dest_vcpu = dst[i]->vcpu; in kvm_intr_is_single_vcpu_fast()
1036 struct kvm_vcpu *vcpu = apic->vcpu; in __apic_accept_irq()
1038 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, in __apic_accept_irq()
1042 vcpu->arch.apic_arb_prio++; in __apic_accept_irq()
1054 __set_bit(vcpu->vcpu_id, dest_map->map); in __apic_accept_irq()
1055 dest_map->vectors[vcpu->vcpu_id] = vector; in __apic_accept_irq()
1058 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { in __apic_accept_irq()
1060 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR); in __apic_accept_irq()
1062 apic_clear_vector(vector, apic->regs + APIC_TMR); in __apic_accept_irq()
1065 if (kvm_x86_ops->deliver_posted_interrupt(vcpu, vector)) { in __apic_accept_irq()
1074 vcpu->arch.pv.pv_unhalted = 1; in __apic_accept_irq()
1095 apic->pending_events = (1UL << KVM_APIC_INIT); in __apic_accept_irq()
1102 apic_debug("Ignoring de-assert INIT to vcpu %d\n", in __apic_accept_irq()
1103 vcpu->vcpu_id); in __apic_accept_irq()
1109 vcpu->vcpu_id, vector); in __apic_accept_irq()
1111 apic->sipi_vector = vector; in __apic_accept_irq()
1114 set_bit(KVM_APIC_SIPI, &apic->pending_events); in __apic_accept_irq()
1137 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; in kvm_apic_compare_prio()
1142 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); in kvm_ioapic_handles_vector()
1154 if (irqchip_split(apic->vcpu->kvm)) { in kvm_ioapic_send_eoi()
1155 apic->vcpu->arch.pending_ioapic_eoi = vector; in kvm_ioapic_send_eoi()
1156 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); in kvm_ioapic_send_eoi()
1160 if (apic_test_vector(vector, apic->regs + APIC_TMR)) in kvm_ioapic_send_eoi()
1165 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); in kvm_ioapic_send_eoi()
1178 if (vector == -1) in apic_set_eoi()
1184 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap)) in apic_set_eoi()
1185 kvm_hv_synic_send_eoi(apic->vcpu, vector); in apic_set_eoi()
1188 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_set_eoi()
1193 * this interface assumes a trap-like exit, which has already finished
1198 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_eoi_accelerated()
1203 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in kvm_apic_set_eoi_accelerated()
1235 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); in apic_send_ipi()
1248 apic->lapic_timer.period == 0) in apic_get_tmcct()
1252 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in apic_get_tmcct()
1256 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); in apic_get_tmcct()
1258 (APIC_BUS_CYCLE_NS * apic->divide_count)); in apic_get_tmcct()
1265 struct kvm_vcpu *vcpu = apic->vcpu; in __report_tpr_access()
1266 struct kvm_run *run = vcpu->run; in __report_tpr_access()
1269 run->tpr_access.rip = kvm_rip_read(vcpu); in __report_tpr_access()
1270 run->tpr_access.is_write = write; in __report_tpr_access()
1275 if (apic->vcpu->arch.tpr_access_reporting) in report_tpr_access()
1322 /* this bitmask has a bit cleared for each reserved register */ in kvm_lapic_reg_read()
1332 apic_debug("KVM_APIC_READ: read reserved register %x\n", in kvm_lapic_reg_read()
1358 return addr >= apic->base_address && in apic_mmio_in_range()
1359 addr < apic->base_address + LAPIC_MMIO_LENGTH; in apic_mmio_in_range()
1366 u32 offset = address - apic->base_address; in apic_mmio_read()
1369 return -EOPNOTSUPP; in apic_mmio_read()
1372 if (!kvm_check_has_quirk(vcpu->kvm, in apic_mmio_read()
1374 return -EOPNOTSUPP; in apic_mmio_read()
1392 apic->divide_count = 0x1 << (tmp2 & 0x7); in update_divide_count()
1395 apic->divide_count); in update_divide_count()
1405 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in limit_periodic_timer_frequency()
1408 if (apic->lapic_timer.period < min_period) { in limit_periodic_timer_frequency()
1412 apic->vcpu->vcpu_id, in limit_periodic_timer_frequency()
1413 apic->lapic_timer.period, min_period); in limit_periodic_timer_frequency()
1414 apic->lapic_timer.period = min_period; in limit_periodic_timer_frequency()
1422 apic->lapic_timer.timer_mode_mask; in apic_update_lvtt()
1424 if (apic->lapic_timer.timer_mode != timer_mode) { in apic_update_lvtt()
1427 hrtimer_cancel(&apic->lapic_timer.timer); in apic_update_lvtt()
1429 apic->lapic_timer.period = 0; in apic_update_lvtt()
1430 apic->lapic_timer.tscdeadline = 0; in apic_update_lvtt()
1432 apic->lapic_timer.timer_mode = timer_mode; in apic_update_lvtt()
1439 struct kvm_vcpu *vcpu = apic->vcpu; in apic_timer_expired()
1440 struct swait_queue_head *q = &vcpu->wq; in apic_timer_expired()
1441 struct kvm_timer *ktimer = &apic->lapic_timer; in apic_timer_expired()
1443 if (atomic_read(&apic->lapic_timer.pending)) in apic_timer_expired()
1446 atomic_inc(&apic->lapic_timer.pending); in apic_timer_expired()
1456 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) in apic_timer_expired()
1457 ktimer->expired_tscdeadline = ktimer->tscdeadline; in apic_timer_expired()
1462 * during a higher-priority task.
1467 struct kvm_lapic *apic = vcpu->arch.apic; in lapic_timer_int_injected()
1472 void *bitmap = apic->regs + APIC_ISR; in lapic_timer_int_injected()
1474 if (vcpu->arch.apicv_active) in lapic_timer_int_injected()
1475 bitmap = apic->regs + APIC_IRR; in lapic_timer_int_injected()
1485 struct kvm_lapic *apic = vcpu->arch.apic; in wait_lapic_expire()
1491 if (apic->lapic_timer.expired_tscdeadline == 0) in wait_lapic_expire()
1497 tsc_deadline = apic->lapic_timer.expired_tscdeadline; in wait_lapic_expire()
1498 apic->lapic_timer.expired_tscdeadline = 0; in wait_lapic_expire()
1500 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); in wait_lapic_expire()
1504 __delay(min(tsc_deadline - guest_tsc, in wait_lapic_expire()
1510 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; in start_sw_tscdeadline()
1513 struct kvm_vcpu *vcpu = apic->vcpu; in start_sw_tscdeadline()
1514 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; in start_sw_tscdeadline()
1526 ns = (tscdeadline - guest_tsc) * 1000000ULL; in start_sw_tscdeadline()
1530 hrtimer_start(&apic->lapic_timer.timer, in start_sw_tscdeadline()
1543 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) in update_target_expiration()
1544 * APIC_BUS_CYCLE_NS * apic->divide_count; in update_target_expiration()
1548 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in update_target_expiration()
1554 apic->divide_count, old_divisor); in update_target_expiration()
1556 apic->lapic_timer.tscdeadline += in update_target_expiration()
1557 nsec_to_cycles(apic->vcpu, ns_remaining_new) - in update_target_expiration()
1558 nsec_to_cycles(apic->vcpu, ns_remaining_old); in update_target_expiration()
1559 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); in update_target_expiration()
1568 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) in set_target_expiration()
1569 * APIC_BUS_CYCLE_NS * apic->divide_count; in set_target_expiration()
1571 if (!apic->lapic_timer.period) { in set_target_expiration()
1572 apic->lapic_timer.tscdeadline = 0; in set_target_expiration()
1584 apic->lapic_timer.period, in set_target_expiration()
1586 apic->lapic_timer.period))); in set_target_expiration()
1588 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in set_target_expiration()
1589 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); in set_target_expiration()
1590 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); in set_target_expiration()
1608 apic->lapic_timer.target_expiration = in advance_periodic_target_expiration()
1609 ktime_add_ns(apic->lapic_timer.target_expiration, in advance_periodic_target_expiration()
1610 apic->lapic_timer.period); in advance_periodic_target_expiration()
1611 delta = ktime_sub(apic->lapic_timer.target_expiration, now); in advance_periodic_target_expiration()
1612 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in advance_periodic_target_expiration()
1613 nsec_to_cycles(apic->vcpu, delta); in advance_periodic_target_expiration()
1618 if (!apic->lapic_timer.period) in start_sw_period()
1622 apic->lapic_timer.target_expiration)) { in start_sw_period()
1631 hrtimer_start(&apic->lapic_timer.timer, in start_sw_period()
1632 apic->lapic_timer.target_expiration, in start_sw_period()
1641 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; in kvm_lapic_hv_timer_in_use()
1648 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in cancel_hv_timer()
1649 kvm_x86_ops->cancel_hv_timer(apic->vcpu); in cancel_hv_timer()
1650 apic->lapic_timer.hv_timer_in_use = false; in cancel_hv_timer()
1655 struct kvm_timer *ktimer = &apic->lapic_timer; in start_hv_timer()
1659 if (!kvm_x86_ops->set_hv_timer) in start_hv_timer()
1662 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) in start_hv_timer()
1665 if (!ktimer->tscdeadline) in start_hv_timer()
1668 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline); in start_hv_timer()
1672 ktimer->hv_timer_in_use = true; in start_hv_timer()
1673 hrtimer_cancel(&ktimer->timer); in start_hv_timer()
1676 * Also recheck ktimer->pending, in case the sw timer triggered in in start_hv_timer()
1680 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) { in start_hv_timer()
1686 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true); in start_hv_timer()
1692 struct kvm_timer *ktimer = &apic->lapic_timer; in start_sw_timer()
1695 if (apic->lapic_timer.hv_timer_in_use) in start_sw_timer()
1697 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) in start_sw_timer()
1704 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); in start_sw_timer()
1717 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_expired_hv_timer()
1721 if (!apic->lapic_timer.hv_timer_in_use) in kvm_lapic_expired_hv_timer()
1723 WARN_ON(swait_active(&vcpu->wq)); in kvm_lapic_expired_hv_timer()
1727 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in kvm_lapic_expired_hv_timer()
1738 restart_apic_timer(vcpu->arch.apic); in kvm_lapic_switch_to_hv_timer()
1744 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_switch_to_sw_timer()
1748 if (apic->lapic_timer.hv_timer_in_use) in kvm_lapic_switch_to_sw_timer()
1756 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_restart_hv_timer()
1758 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in kvm_lapic_restart_hv_timer()
1764 atomic_set(&apic->lapic_timer.pending, 0); in start_apic_timer()
1777 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { in apic_manage_nmi_watchdog()
1778 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; in apic_manage_nmi_watchdog()
1781 "for cpu %d\n", apic->vcpu->vcpu_id); in apic_manage_nmi_watchdog()
1782 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
1784 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
1821 recalculate_apic_map(apic->vcpu->kvm); in kvm_lapic_reg_write()
1842 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reg_write()
1873 (reg - APIC_LVTT) >> 4, size); in kvm_lapic_reg_write()
1882 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); in kvm_lapic_reg_write()
1891 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reg_write()
1897 uint32_t old_divisor = apic->divide_count; in kvm_lapic_reg_write()
1903 if (apic->divide_count != old_divisor && in kvm_lapic_reg_write()
1904 apic->lapic_timer.period) { in kvm_lapic_reg_write()
1905 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reg_write()
1929 apic_debug("Local APIC Write to read-only register %x\n", reg); in kvm_lapic_reg_write()
1938 unsigned int offset = address - apic->base_address; in apic_mmio_write()
1942 return -EOPNOTSUPP; in apic_mmio_write()
1945 if (!kvm_check_has_quirk(vcpu->kvm, in apic_mmio_write()
1947 return -EOPNOTSUPP; in apic_mmio_write()
1953 * APIC register must be aligned on 128-bits boundary. in apic_mmio_write()
1977 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); in kvm_lapic_set_eoi()
1989 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val); in kvm_apic_write_nodecode()
1992 kvm_lapic_reg_write(vcpu->arch.apic, offset, val); in kvm_apic_write_nodecode()
1998 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_free_lapic()
2000 if (!vcpu->arch.apic) in kvm_free_lapic()
2003 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_free_lapic()
2005 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) in kvm_free_lapic()
2008 if (!apic->sw_enabled) in kvm_free_lapic()
2011 if (apic->regs) in kvm_free_lapic()
2012 free_page((unsigned long)apic->regs); in kvm_free_lapic()
2018 *----------------------------------------------------------------------
2020 *----------------------------------------------------------------------
2024 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_get_lapic_tscdeadline_msr()
2030 return apic->lapic_timer.tscdeadline; in kvm_get_lapic_tscdeadline_msr()
2035 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_set_lapic_tscdeadline_msr()
2041 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_set_lapic_tscdeadline_msr()
2042 apic->lapic_timer.tscdeadline = data; in kvm_set_lapic_tscdeadline_msr()
2048 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_set_tpr()
2058 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); in kvm_lapic_get_cr8()
2065 u64 old_value = vcpu->arch.apic_base; in kvm_lapic_set_base()
2066 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_set_base()
2071 vcpu->arch.apic_base = value; in kvm_lapic_set_base()
2082 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in kvm_lapic_set_base()
2086 recalculate_apic_map(vcpu->kvm); in kvm_lapic_set_base()
2091 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); in kvm_lapic_set_base()
2094 kvm_x86_ops->set_virtual_apic_mode(vcpu); in kvm_lapic_set_base()
2096 apic->base_address = apic->vcpu->arch.apic_base & in kvm_lapic_set_base()
2100 apic->base_address != APIC_DEFAULT_PHYS_BASE) in kvm_lapic_set_base()
2105 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); in kvm_lapic_set_base()
2111 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_reset()
2120 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reset()
2125 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in kvm_lapic_reset()
2127 kvm_apic_set_version(apic->vcpu); in kvm_lapic_reset()
2133 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) in kvm_lapic_reset()
2153 apic->irr_pending = vcpu->arch.apicv_active; in kvm_lapic_reset()
2154 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0; in kvm_lapic_reset()
2155 apic->highest_isr_cache = -1; in kvm_lapic_reset()
2157 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reset()
2160 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); in kvm_lapic_reset()
2161 vcpu->arch.pv_eoi.msr_val = 0; in kvm_lapic_reset()
2163 if (vcpu->arch.apicv_active) { in kvm_lapic_reset()
2164 kvm_x86_ops->apicv_post_state_restore(vcpu); in kvm_lapic_reset()
2165 kvm_x86_ops->hwapic_irr_update(vcpu, -1); in kvm_lapic_reset()
2166 kvm_x86_ops->hwapic_isr_update(vcpu, -1); in kvm_lapic_reset()
2169 vcpu->arch.apic_arb_prio = 0; in kvm_lapic_reset()
2170 vcpu->arch.apic_attention = 0; in kvm_lapic_reset()
2175 vcpu->arch.apic_base, apic->base_address); in kvm_lapic_reset()
2179 *----------------------------------------------------------------------
2181 *----------------------------------------------------------------------
2191 struct kvm_lapic *apic = vcpu->arch.apic; in apic_has_pending_timer()
2194 return atomic_read(&apic->lapic_timer.pending); in apic_has_pending_timer()
2216 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_nmi_wd_deliver()
2236 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); in apic_timer_fn()
2247 apic_debug("apic_init %d\n", vcpu->vcpu_id); in kvm_create_lapic()
2253 vcpu->arch.apic = apic; in kvm_create_lapic()
2255 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); in kvm_create_lapic()
2256 if (!apic->regs) { in kvm_create_lapic()
2258 vcpu->vcpu_id); in kvm_create_lapic()
2261 apic->vcpu = vcpu; in kvm_create_lapic()
2263 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, in kvm_create_lapic()
2265 apic->lapic_timer.timer.function = apic_timer_fn; in kvm_create_lapic()
2271 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; in kvm_create_lapic()
2273 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); in kvm_create_lapic()
2279 return -ENOMEM; in kvm_create_lapic()
2284 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_has_interrupt()
2288 return -1; in kvm_apic_has_interrupt()
2296 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); in kvm_apic_accept_pic_intr()
2299 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) in kvm_apic_accept_pic_intr()
2309 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_inject_apic_timer_irqs()
2311 if (atomic_read(&apic->lapic_timer.pending) > 0) { in kvm_inject_apic_timer_irqs()
2314 apic->lapic_timer.tscdeadline = 0; in kvm_inject_apic_timer_irqs()
2316 apic->lapic_timer.tscdeadline = 0; in kvm_inject_apic_timer_irqs()
2317 apic->lapic_timer.target_expiration = 0; in kvm_inject_apic_timer_irqs()
2319 atomic_set(&apic->lapic_timer.pending, 0); in kvm_inject_apic_timer_irqs()
2326 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_get_apic_interrupt()
2329 if (vector == -1) in kvm_get_apic_interrupt()
2330 return -1; in kvm_get_apic_interrupt()
2340 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) { in kvm_get_apic_interrupt()
2342 * For auto-EOI interrupts, there might be another pending in kvm_get_apic_interrupt()
2350 * be a higher-priority pending interrupt---except if there was in kvm_get_apic_interrupt()
2364 if (apic_x2apic_mode(vcpu->arch.apic)) { in kvm_apic_state_fixup()
2365 u32 *id = (u32 *)(s->regs + APIC_ID); in kvm_apic_state_fixup()
2366 u32 *ldr = (u32 *)(s->regs + APIC_LDR); in kvm_apic_state_fixup()
2368 if (vcpu->kvm->arch.x2apic_format) { in kvm_apic_state_fixup()
2369 if (*id != vcpu->vcpu_id) in kvm_apic_state_fixup()
2370 return -EINVAL; in kvm_apic_state_fixup()
2388 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); in kvm_apic_get_state()
2394 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_state()
2398 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); in kvm_apic_set_state()
2400 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); in kvm_apic_set_state()
2405 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); in kvm_apic_set_state()
2407 recalculate_apic_map(vcpu->kvm); in kvm_apic_set_state()
2411 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_apic_set_state()
2416 apic->irr_pending = true; in kvm_apic_set_state()
2417 apic->isr_count = vcpu->arch.apicv_active ? in kvm_apic_set_state()
2418 1 : count_vectors(apic->regs + APIC_ISR); in kvm_apic_set_state()
2419 apic->highest_isr_cache = -1; in kvm_apic_set_state()
2420 if (vcpu->arch.apicv_active) { in kvm_apic_set_state()
2421 kvm_x86_ops->apicv_post_state_restore(vcpu); in kvm_apic_set_state()
2422 kvm_x86_ops->hwapic_irr_update(vcpu, in kvm_apic_set_state()
2424 kvm_x86_ops->hwapic_isr_update(vcpu, in kvm_apic_set_state()
2428 if (ioapic_in_kernel(vcpu->kvm)) in kvm_apic_set_state()
2431 vcpu->arch.apic_arb_prio = 0; in kvm_apic_set_state()
2443 timer = &vcpu->arch.apic->lapic_timer.timer; in __kvm_migrate_apic_timer()
2449 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2465 * -> host disabled PV EOI. in apic_sync_pv_eoi_from_guest()
2467 * -> host enabled PV EOI, guest did not execute EOI yet. in apic_sync_pv_eoi_from_guest()
2469 * -> host enabled PV EOI, guest executed EOI. in apic_sync_pv_eoi_from_guest()
2489 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) in kvm_lapic_sync_from_vapic()
2490 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); in kvm_lapic_sync_from_vapic()
2492 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) in kvm_lapic_sync_from_vapic()
2495 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_from_vapic()
2499 apic_set_tpr(vcpu->arch.apic, data & 0xff); in kvm_lapic_sync_from_vapic()
2503 * apic_sync_pv_eoi_to_guest - called before vmentry
2513 apic->irr_pending || in apic_sync_pv_eoi_to_guest()
2515 apic->highest_isr_cache == -1 || in apic_sync_pv_eoi_to_guest()
2517 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { in apic_sync_pv_eoi_to_guest()
2525 pv_eoi_set_pending(apic->vcpu); in apic_sync_pv_eoi_to_guest()
2532 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_sync_to_vapic()
2536 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) in kvm_lapic_sync_to_vapic()
2548 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_to_vapic()
2555 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, in kvm_lapic_set_vapic_addr()
2556 &vcpu->arch.apic->vapic_cache, in kvm_lapic_set_vapic_addr()
2558 return -EINVAL; in kvm_lapic_set_vapic_addr()
2559 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); in kvm_lapic_set_vapic_addr()
2561 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); in kvm_lapic_set_vapic_addr()
2564 vcpu->arch.apic->vapic_addr = vapic_addr; in kvm_lapic_set_vapic_addr()
2570 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_write()
2571 u32 reg = (msr - APIC_BASE_MSR) << 4; in kvm_x2apic_msr_write()
2587 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_read()
2588 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; in kvm_x2apic_msr_read()
2594 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", in kvm_x2apic_msr_read()
2611 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_hv_vapic_msr_write()
2624 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_hv_vapic_msr_read()
2643 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; in kvm_lapic_enable_pv_eoi()
2649 vcpu->arch.pv_eoi.msr_val = data; in kvm_lapic_enable_pv_eoi()
2653 if (addr == ghc->gpa && len <= ghc->len) in kvm_lapic_enable_pv_eoi()
2654 new_len = ghc->len; in kvm_lapic_enable_pv_eoi()
2658 return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); in kvm_lapic_enable_pv_eoi()
2663 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_accept_events()
2667 if (!lapic_in_kernel(vcpu) || !apic->pending_events) in kvm_apic_accept_events()
2676 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); in kvm_apic_accept_events()
2677 if (test_bit(KVM_APIC_SIPI, &apic->pending_events)) in kvm_apic_accept_events()
2678 clear_bit(KVM_APIC_SIPI, &apic->pending_events); in kvm_apic_accept_events()
2682 pe = xchg(&apic->pending_events, 0); in kvm_apic_accept_events()
2685 if (kvm_vcpu_is_bsp(apic->vcpu)) in kvm_apic_accept_events()
2686 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; in kvm_apic_accept_events()
2688 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; in kvm_apic_accept_events()
2691 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { in kvm_apic_accept_events()
2694 sipi_vector = apic->sipi_vector; in kvm_apic_accept_events()
2696 vcpu->vcpu_id, sipi_vector); in kvm_apic_accept_events()
2698 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; in kvm_apic_accept_events()