1 /*
2 * Local APIC handling, local APIC timers
3 *
4 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
5 *
6 * Fixes
7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
8 * thanks to Eric Gilmore
9 * and Rolf G. Tews
10 * for testing these extensively.
11 * Maciej W. Rozycki : Various updates and fixes.
12 * Mikael Pettersson : Power Management for UP-APIC.
13 * Pavel Machek and
14 * Mikael Pettersson : PM converted to driver model.
15 */
16
17 #include <linux/perf_event.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/mc146818rtc.h>
20 #include <linux/acpi_pmtmr.h>
21 #include <linux/clockchips.h>
22 #include <linux/interrupt.h>
23 #include <linux/bootmem.h>
24 #include <linux/ftrace.h>
25 #include <linux/ioport.h>
26 #include <linux/module.h>
27 #include <linux/syscore_ops.h>
28 #include <linux/delay.h>
29 #include <linux/timex.h>
30 #include <linux/i8253.h>
31 #include <linux/dmar.h>
32 #include <linux/init.h>
33 #include <linux/cpu.h>
34 #include <linux/dmi.h>
35 #include <linux/smp.h>
36 #include <linux/mm.h>
37
38 #include <asm/trace/irq_vectors.h>
39 #include <asm/irq_remapping.h>
40 #include <asm/perf_event.h>
41 #include <asm/x86_init.h>
42 #include <asm/pgalloc.h>
43 #include <linux/atomic.h>
44 #include <asm/barrier.h>
45 #include <asm/mpspec.h>
46 #include <asm/i8259.h>
47 #include <asm/proto.h>
48 #include <asm/apic.h>
49 #include <asm/io_apic.h>
50 #include <asm/desc.h>
51 #include <asm/hpet.h>
52 #include <asm/idle.h>
53 #include <asm/mtrr.h>
54 #include <asm/time.h>
55 #include <asm/smp.h>
56 #include <asm/mce.h>
57 #include <asm/tsc.h>
58 #include <asm/hypervisor.h>
59
60 unsigned int num_processors;
61
62 unsigned disabled_cpus;
63
64 /* Processor that is doing the boot up */
65 unsigned int boot_cpu_physical_apicid = -1U;
66 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
67
68 /*
69 * The highest APIC ID seen during enumeration.
70 */
71 static unsigned int max_physical_apicid;
72
73 /*
74 * Bitmask of physically existing CPUs:
75 */
76 physid_mask_t phys_cpu_present_map;
77
78 /*
79 * Processor to be disabled specified by kernel parameter
80 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
81 * avoid undefined behaviour caused by sending INIT from AP to BSP.
82 */
83 static unsigned int disabled_cpu_apicid __read_mostly = BAD_APICID;
84
85 /*
86 * Map cpu index to physical APIC ID
87 */
88 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
89 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
90 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
91 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
92
93 #ifdef CONFIG_X86_32
94
95 /*
96 * On x86_32, the mapping between cpu and logical apicid may vary
97 * depending on apic in use. The following early percpu variable is
98 * used for the mapping. This is where the behaviors of x86_64 and 32
99 * actually diverge. Let's keep it ugly for now.
100 */
101 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
102
103 /* Local APIC was disabled by the BIOS and enabled by the kernel */
104 static int enabled_via_apicbase;
105
106 /*
107 * Handle interrupt mode configuration register (IMCR).
108 * This register controls whether the interrupt signals
109 * that reach the BSP come from the master PIC or from the
110 * local APIC. Before entering Symmetric I/O Mode, either
111 * the BIOS or the operating system must switch out of
112 * PIC Mode by changing the IMCR.
113 */
imcr_pic_to_apic(void)114 static inline void imcr_pic_to_apic(void)
115 {
116 /* select IMCR register */
117 outb(0x70, 0x22);
118 /* NMI and 8259 INTR go through APIC */
119 outb(0x01, 0x23);
120 }
121
imcr_apic_to_pic(void)122 static inline void imcr_apic_to_pic(void)
123 {
124 /* select IMCR register */
125 outb(0x70, 0x22);
126 /* NMI and 8259 INTR go directly to BSP */
127 outb(0x00, 0x23);
128 }
129 #endif
130
131 /*
132 * Knob to control our willingness to enable the local APIC.
133 *
134 * +1=force-enable
135 */
136 static int force_enable_local_apic __initdata;
137
138 /*
139 * APIC command line parameters
140 */
parse_lapic(char * arg)141 static int __init parse_lapic(char *arg)
142 {
143 if (config_enabled(CONFIG_X86_32) && !arg)
144 force_enable_local_apic = 1;
145 else if (arg && !strncmp(arg, "notscdeadline", 13))
146 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
147 return 0;
148 }
149 early_param("lapic", parse_lapic);
150
151 #ifdef CONFIG_X86_64
152 static int apic_calibrate_pmtmr __initdata;
setup_apicpmtimer(char * s)153 static __init int setup_apicpmtimer(char *s)
154 {
155 apic_calibrate_pmtmr = 1;
156 notsc_setup(NULL);
157 return 0;
158 }
159 __setup("apicpmtimer", setup_apicpmtimer);
160 #endif
161
162 unsigned long mp_lapic_addr;
163 int disable_apic;
164 /* Disable local APIC timer from the kernel commandline or via dmi quirk */
165 static int disable_apic_timer __initdata;
166 /* Local APIC timer works in C2 */
167 int local_apic_timer_c2_ok;
168 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
169
170 int first_system_vector = FIRST_SYSTEM_VECTOR;
171
172 /*
173 * Debug level, exported for io_apic.c
174 */
175 int apic_verbosity;
176
177 int pic_mode;
178
179 /* Have we found an MP table */
180 int smp_found_config;
181
182 static struct resource lapic_resource = {
183 .name = "Local APIC",
184 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
185 };
186
187 unsigned int lapic_timer_frequency = 0;
188
189 static void apic_pm_activate(void);
190
191 static unsigned long apic_phys;
192
193 /*
194 * Get the LAPIC version
195 */
lapic_get_version(void)196 static inline int lapic_get_version(void)
197 {
198 return GET_APIC_VERSION(apic_read(APIC_LVR));
199 }
200
201 /*
202 * Check, if the APIC is integrated or a separate chip
203 */
lapic_is_integrated(void)204 static inline int lapic_is_integrated(void)
205 {
206 #ifdef CONFIG_X86_64
207 return 1;
208 #else
209 return APIC_INTEGRATED(lapic_get_version());
210 #endif
211 }
212
213 /*
214 * Check, whether this is a modern or a first generation APIC
215 */
modern_apic(void)216 static int modern_apic(void)
217 {
218 /* AMD systems use old APIC versions, so check the CPU */
219 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
220 boot_cpu_data.x86 >= 0xf)
221 return 1;
222 return lapic_get_version() >= 0x14;
223 }
224
225 /*
226 * right after this call apic become NOOP driven
227 * so apic->write/read doesn't do anything
228 */
apic_disable(void)229 static void __init apic_disable(void)
230 {
231 pr_info("APIC: switched to apic NOOP\n");
232 apic = &apic_noop;
233 }
234
native_apic_wait_icr_idle(void)235 void native_apic_wait_icr_idle(void)
236 {
237 while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
238 cpu_relax();
239 }
240
native_safe_apic_wait_icr_idle(void)241 u32 native_safe_apic_wait_icr_idle(void)
242 {
243 u32 send_status;
244 int timeout;
245
246 timeout = 0;
247 do {
248 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
249 if (!send_status)
250 break;
251 inc_irq_stat(icr_read_retry_count);
252 udelay(100);
253 } while (timeout++ < 1000);
254
255 return send_status;
256 }
257
native_apic_icr_write(u32 low,u32 id)258 void native_apic_icr_write(u32 low, u32 id)
259 {
260 unsigned long flags;
261
262 local_irq_save(flags);
263 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
264 apic_write(APIC_ICR, low);
265 local_irq_restore(flags);
266 }
267
native_apic_icr_read(void)268 u64 native_apic_icr_read(void)
269 {
270 u32 icr1, icr2;
271
272 icr2 = apic_read(APIC_ICR2);
273 icr1 = apic_read(APIC_ICR);
274
275 return icr1 | ((u64)icr2 << 32);
276 }
277
278 #ifdef CONFIG_X86_32
279 /**
280 * get_physical_broadcast - Get number of physical broadcast IDs
281 */
get_physical_broadcast(void)282 int get_physical_broadcast(void)
283 {
284 return modern_apic() ? 0xff : 0xf;
285 }
286 #endif
287
288 /**
289 * lapic_get_maxlvt - get the maximum number of local vector table entries
290 */
lapic_get_maxlvt(void)291 int lapic_get_maxlvt(void)
292 {
293 unsigned int v;
294
295 v = apic_read(APIC_LVR);
296 /*
297 * - we always have APIC integrated on 64bit mode
298 * - 82489DXs do not report # of LVT entries
299 */
300 return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
301 }
302
303 /*
304 * Local APIC timer
305 */
306
307 /* Clock divisor */
308 #define APIC_DIVISOR 16
309 #define TSC_DIVISOR 32
310
311 /*
312 * This function sets up the local APIC timer, with a timeout of
313 * 'clocks' APIC bus clock. During calibration we actually call
314 * this function twice on the boot CPU, once with a bogus timeout
315 * value, second time for real. The other (noncalibrating) CPUs
316 * call this function only once, with the real, calibrated value.
317 *
318 * We do reads before writes even if unnecessary, to get around the
319 * P5 APIC double write bug.
320 */
__setup_APIC_LVTT(unsigned int clocks,int oneshot,int irqen)321 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
322 {
323 unsigned int lvtt_value, tmp_value;
324
325 lvtt_value = LOCAL_TIMER_VECTOR;
326 if (!oneshot)
327 lvtt_value |= APIC_LVT_TIMER_PERIODIC;
328 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
329 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
330
331 if (!lapic_is_integrated())
332 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
333
334 if (!irqen)
335 lvtt_value |= APIC_LVT_MASKED;
336
337 apic_write(APIC_LVTT, lvtt_value);
338
339 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
340 /*
341 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
342 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
343 * According to Intel, MFENCE can do the serialization here.
344 */
345 asm volatile("mfence" : : : "memory");
346
347 printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
348 return;
349 }
350
351 /*
352 * Divide PICLK by 16
353 */
354 tmp_value = apic_read(APIC_TDCR);
355 apic_write(APIC_TDCR,
356 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
357 APIC_TDR_DIV_16);
358
359 if (!oneshot)
360 apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
361 }
362
363 /*
364 * Setup extended LVT, AMD specific
365 *
366 * Software should use the LVT offsets the BIOS provides. The offsets
367 * are determined by the subsystems using it like those for MCE
368 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
369 * are supported. Beginning with family 10h at least 4 offsets are
370 * available.
371 *
372 * Since the offsets must be consistent for all cores, we keep track
373 * of the LVT offsets in software and reserve the offset for the same
374 * vector also to be used on other cores. An offset is freed by
375 * setting the entry to APIC_EILVT_MASKED.
376 *
377 * If the BIOS is right, there should be no conflicts. Otherwise a
378 * "[Firmware Bug]: ..." error message is generated. However, if
379 * software does not properly determines the offsets, it is not
380 * necessarily a BIOS bug.
381 */
382
383 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
384
eilvt_entry_is_changeable(unsigned int old,unsigned int new)385 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
386 {
387 return (old & APIC_EILVT_MASKED)
388 || (new == APIC_EILVT_MASKED)
389 || ((new & ~APIC_EILVT_MASKED) == old);
390 }
391
reserve_eilvt_offset(int offset,unsigned int new)392 static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
393 {
394 unsigned int rsvd, vector;
395
396 if (offset >= APIC_EILVT_NR_MAX)
397 return ~0;
398
399 rsvd = atomic_read(&eilvt_offsets[offset]);
400 do {
401 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
402 if (vector && !eilvt_entry_is_changeable(vector, new))
403 /* may not change if vectors are different */
404 return rsvd;
405 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
406 } while (rsvd != new);
407
408 rsvd &= ~APIC_EILVT_MASKED;
409 if (rsvd && rsvd != vector)
410 pr_info("LVT offset %d assigned for vector 0x%02x\n",
411 offset, rsvd);
412
413 return new;
414 }
415
416 /*
417 * If mask=1, the LVT entry does not generate interrupts while mask=0
418 * enables the vector. See also the BKDGs. Must be called with
419 * preemption disabled.
420 */
421
setup_APIC_eilvt(u8 offset,u8 vector,u8 msg_type,u8 mask)422 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
423 {
424 unsigned long reg = APIC_EILVTn(offset);
425 unsigned int new, old, reserved;
426
427 new = (mask << 16) | (msg_type << 8) | vector;
428 old = apic_read(reg);
429 reserved = reserve_eilvt_offset(offset, new);
430
431 if (reserved != new) {
432 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
433 "vector 0x%x, but the register is already in use for "
434 "vector 0x%x on another cpu\n",
435 smp_processor_id(), reg, offset, new, reserved);
436 return -EINVAL;
437 }
438
439 if (!eilvt_entry_is_changeable(old, new)) {
440 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
441 "vector 0x%x, but the register is already in use for "
442 "vector 0x%x on this cpu\n",
443 smp_processor_id(), reg, offset, new, old);
444 return -EBUSY;
445 }
446
447 apic_write(reg, new);
448
449 return 0;
450 }
451 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
452
453 /*
454 * Program the next event, relative to now
455 */
lapic_next_event(unsigned long delta,struct clock_event_device * evt)456 static int lapic_next_event(unsigned long delta,
457 struct clock_event_device *evt)
458 {
459 apic_write(APIC_TMICT, delta);
460 return 0;
461 }
462
lapic_next_deadline(unsigned long delta,struct clock_event_device * evt)463 static int lapic_next_deadline(unsigned long delta,
464 struct clock_event_device *evt)
465 {
466 u64 tsc;
467
468 /* This MSR is special and need a special fence: */
469 weak_wrmsr_fence();
470
471 tsc = rdtsc();
472 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
473 return 0;
474 }
475
lapic_timer_shutdown(struct clock_event_device * evt)476 static int lapic_timer_shutdown(struct clock_event_device *evt)
477 {
478 unsigned int v;
479
480 /* Lapic used as dummy for broadcast ? */
481 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
482 return 0;
483
484 v = apic_read(APIC_LVTT);
485 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
486 apic_write(APIC_LVTT, v);
487 apic_write(APIC_TMICT, 0);
488 return 0;
489 }
490
491 static inline int
lapic_timer_set_periodic_oneshot(struct clock_event_device * evt,bool oneshot)492 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
493 {
494 /* Lapic used as dummy for broadcast ? */
495 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
496 return 0;
497
498 __setup_APIC_LVTT(lapic_timer_frequency, oneshot, 1);
499 return 0;
500 }
501
lapic_timer_set_periodic(struct clock_event_device * evt)502 static int lapic_timer_set_periodic(struct clock_event_device *evt)
503 {
504 return lapic_timer_set_periodic_oneshot(evt, false);
505 }
506
lapic_timer_set_oneshot(struct clock_event_device * evt)507 static int lapic_timer_set_oneshot(struct clock_event_device *evt)
508 {
509 return lapic_timer_set_periodic_oneshot(evt, true);
510 }
511
512 /*
513 * Local APIC timer broadcast function
514 */
lapic_timer_broadcast(const struct cpumask * mask)515 static void lapic_timer_broadcast(const struct cpumask *mask)
516 {
517 #ifdef CONFIG_SMP
518 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
519 #endif
520 }
521
522
523 /*
524 * The local apic timer can be used for any function which is CPU local.
525 */
526 static struct clock_event_device lapic_clockevent = {
527 .name = "lapic",
528 .features = CLOCK_EVT_FEAT_PERIODIC |
529 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
530 | CLOCK_EVT_FEAT_DUMMY,
531 .shift = 32,
532 .set_state_shutdown = lapic_timer_shutdown,
533 .set_state_periodic = lapic_timer_set_periodic,
534 .set_state_oneshot = lapic_timer_set_oneshot,
535 .set_next_event = lapic_next_event,
536 .broadcast = lapic_timer_broadcast,
537 .rating = 100,
538 .irq = -1,
539 };
540 static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
541
542 /*
543 * Setup the local APIC timer for this CPU. Copy the initialized values
544 * of the boot CPU and register the clock event in the framework.
545 */
setup_APIC_timer(void)546 static void setup_APIC_timer(void)
547 {
548 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
549
550 if (this_cpu_has(X86_FEATURE_ARAT)) {
551 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
552 /* Make LAPIC timer preferrable over percpu HPET */
553 lapic_clockevent.rating = 150;
554 }
555
556 memcpy(levt, &lapic_clockevent, sizeof(*levt));
557 levt->cpumask = cpumask_of(smp_processor_id());
558
559 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
560 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
561 CLOCK_EVT_FEAT_DUMMY);
562 levt->set_next_event = lapic_next_deadline;
563 clockevents_config_and_register(levt,
564 (tsc_khz / TSC_DIVISOR) * 1000,
565 0xF, ~0UL);
566 } else
567 clockevents_register_device(levt);
568 }
569
570 /*
571 * In this functions we calibrate APIC bus clocks to the external timer.
572 *
573 * We want to do the calibration only once since we want to have local timer
574 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
575 * frequency.
576 *
577 * This was previously done by reading the PIT/HPET and waiting for a wrap
578 * around to find out, that a tick has elapsed. I have a box, where the PIT
579 * readout is broken, so it never gets out of the wait loop again. This was
580 * also reported by others.
581 *
582 * Monitoring the jiffies value is inaccurate and the clockevents
583 * infrastructure allows us to do a simple substitution of the interrupt
584 * handler.
585 *
586 * The calibration routine also uses the pm_timer when possible, as the PIT
587 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
588 * back to normal later in the boot process).
589 */
590
591 #define LAPIC_CAL_LOOPS (HZ/10)
592
593 static __initdata int lapic_cal_loops = -1;
594 static __initdata long lapic_cal_t1, lapic_cal_t2;
595 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
596 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
597 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
598
599 /*
600 * Temporary interrupt handler and polled calibration function.
601 */
lapic_cal_handler(struct clock_event_device * dev)602 static void __init lapic_cal_handler(struct clock_event_device *dev)
603 {
604 unsigned long long tsc = 0;
605 long tapic = apic_read(APIC_TMCCT);
606 unsigned long pm = acpi_pm_read_early();
607
608 if (cpu_has_tsc)
609 tsc = rdtsc();
610
611 switch (lapic_cal_loops++) {
612 case 0:
613 lapic_cal_t1 = tapic;
614 lapic_cal_tsc1 = tsc;
615 lapic_cal_pm1 = pm;
616 lapic_cal_j1 = jiffies;
617 break;
618
619 case LAPIC_CAL_LOOPS:
620 lapic_cal_t2 = tapic;
621 lapic_cal_tsc2 = tsc;
622 if (pm < lapic_cal_pm1)
623 pm += ACPI_PM_OVRRUN;
624 lapic_cal_pm2 = pm;
625 lapic_cal_j2 = jiffies;
626 break;
627 }
628 }
629
630 static int __init
calibrate_by_pmtimer(long deltapm,long * delta,long * deltatsc)631 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
632 {
633 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
634 const long pm_thresh = pm_100ms / 100;
635 unsigned long mult;
636 u64 res;
637
638 #ifndef CONFIG_X86_PM_TIMER
639 return -1;
640 #endif
641
642 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
643
644 /* Check, if the PM timer is available */
645 if (!deltapm)
646 return -1;
647
648 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
649
650 if (deltapm > (pm_100ms - pm_thresh) &&
651 deltapm < (pm_100ms + pm_thresh)) {
652 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
653 return 0;
654 }
655
656 res = (((u64)deltapm) * mult) >> 22;
657 do_div(res, 1000000);
658 pr_warning("APIC calibration not consistent "
659 "with PM-Timer: %ldms instead of 100ms\n",(long)res);
660
661 /* Correct the lapic counter value */
662 res = (((u64)(*delta)) * pm_100ms);
663 do_div(res, deltapm);
664 pr_info("APIC delta adjusted to PM-Timer: "
665 "%lu (%ld)\n", (unsigned long)res, *delta);
666 *delta = (long)res;
667
668 /* Correct the tsc counter value */
669 if (cpu_has_tsc) {
670 res = (((u64)(*deltatsc)) * pm_100ms);
671 do_div(res, deltapm);
672 apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
673 "PM-Timer: %lu (%ld)\n",
674 (unsigned long)res, *deltatsc);
675 *deltatsc = (long)res;
676 }
677
678 return 0;
679 }
680
calibrate_APIC_clock(void)681 static int __init calibrate_APIC_clock(void)
682 {
683 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
684 u64 tsc_perj = 0, tsc_start = 0;
685 unsigned long jif_start;
686 unsigned long deltaj;
687 long delta, deltatsc;
688 int pm_referenced = 0;
689
690 /**
691 * check if lapic timer has already been calibrated by platform
692 * specific routine, such as tsc calibration code. if so, we just fill
693 * in the clockevent structure and return.
694 */
695
696 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
697 return 0;
698 } else if (lapic_timer_frequency) {
699 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
700 lapic_timer_frequency);
701 lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
702 TICK_NSEC, lapic_clockevent.shift);
703 lapic_clockevent.max_delta_ns =
704 clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
705 lapic_clockevent.min_delta_ns =
706 clockevent_delta2ns(0xF, &lapic_clockevent);
707 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
708 return 0;
709 }
710
711 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
712 "calibrating APIC timer ...\n");
713
714 /*
715 * There are platforms w/o global clockevent devices. Instead of
716 * making the calibration conditional on that, use a polling based
717 * approach everywhere.
718 */
719 local_irq_disable();
720
721 /*
722 * Setup the APIC counter to maximum. There is no way the lapic
723 * can underflow in the 100ms detection time frame
724 */
725 __setup_APIC_LVTT(0xffffffff, 0, 0);
726
727 /*
728 * Methods to terminate the calibration loop:
729 * 1) Global clockevent if available (jiffies)
730 * 2) TSC if available and frequency is known
731 */
732 jif_start = READ_ONCE(jiffies);
733
734 if (tsc_khz) {
735 tsc_start = rdtsc();
736 tsc_perj = div_u64((u64)tsc_khz * 1000, HZ);
737 }
738
739 /*
740 * Enable interrupts so the tick can fire, if a global
741 * clockevent device is available
742 */
743 local_irq_enable();
744
745 while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
746 /* Wait for a tick to elapse */
747 while (1) {
748 if (tsc_khz) {
749 u64 tsc_now = rdtsc();
750 if ((tsc_now - tsc_start) >= tsc_perj) {
751 tsc_start += tsc_perj;
752 break;
753 }
754 } else {
755 unsigned long jif_now = READ_ONCE(jiffies);
756
757 if (time_after(jif_now, jif_start)) {
758 jif_start = jif_now;
759 break;
760 }
761 }
762 cpu_relax();
763 }
764
765 /* Invoke the calibration routine */
766 local_irq_disable();
767 lapic_cal_handler(NULL);
768 local_irq_enable();
769 }
770
771 local_irq_disable();
772
773 /* Build delta t1-t2 as apic timer counts down */
774 delta = lapic_cal_t1 - lapic_cal_t2;
775 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
776
777 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
778
779 /* we trust the PM based calibration if possible */
780 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
781 &delta, &deltatsc);
782
783 /* Calculate the scaled math multiplication factor */
784 lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
785 lapic_clockevent.shift);
786 lapic_clockevent.max_delta_ns =
787 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
788 lapic_clockevent.min_delta_ns =
789 clockevent_delta2ns(0xF, &lapic_clockevent);
790
791 lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
792
793 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
794 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
795 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
796 lapic_timer_frequency);
797
798 if (cpu_has_tsc) {
799 apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
800 "%ld.%04ld MHz.\n",
801 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
802 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
803 }
804
805 apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
806 "%u.%04u MHz.\n",
807 lapic_timer_frequency / (1000000 / HZ),
808 lapic_timer_frequency % (1000000 / HZ));
809
810 /*
811 * Do a sanity check on the APIC calibration result
812 */
813 if (lapic_timer_frequency < (1000000 / HZ)) {
814 local_irq_enable();
815 pr_warning("APIC frequency too slow, disabling apic timer\n");
816 return -1;
817 }
818
819 levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
820
821 /*
822 * PM timer calibration failed or not turned on so lets try APIC
823 * timer based calibration, if a global clockevent device is
824 * available.
825 */
826 if (!pm_referenced && global_clock_event) {
827 apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
828
829 /*
830 * Setup the apic timer manually
831 */
832 levt->event_handler = lapic_cal_handler;
833 lapic_timer_set_periodic(levt);
834 lapic_cal_loops = -1;
835
836 /* Let the interrupts run */
837 local_irq_enable();
838
839 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
840 cpu_relax();
841
842 /* Stop the lapic timer */
843 local_irq_disable();
844 lapic_timer_shutdown(levt);
845
846 /* Jiffies delta */
847 deltaj = lapic_cal_j2 - lapic_cal_j1;
848 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
849
850 /* Check, if the jiffies result is consistent */
851 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
852 apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
853 else
854 levt->features |= CLOCK_EVT_FEAT_DUMMY;
855 }
856 local_irq_enable();
857
858 if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
859 pr_warning("APIC timer disabled due to verification failure\n");
860 return -1;
861 }
862
863 return 0;
864 }
865
866 /*
867 * Setup the boot APIC
868 *
869 * Calibrate and verify the result.
870 */
setup_boot_APIC_clock(void)871 void __init setup_boot_APIC_clock(void)
872 {
873 /*
874 * The local apic timer can be disabled via the kernel
875 * commandline or from the CPU detection code. Register the lapic
876 * timer as a dummy clock event source on SMP systems, so the
877 * broadcast mechanism is used. On UP systems simply ignore it.
878 */
879 if (disable_apic_timer) {
880 pr_info("Disabling APIC timer\n");
881 /* No broadcast on UP ! */
882 if (num_possible_cpus() > 1) {
883 lapic_clockevent.mult = 1;
884 setup_APIC_timer();
885 }
886 return;
887 }
888
889 if (calibrate_APIC_clock()) {
890 /* No broadcast on UP ! */
891 if (num_possible_cpus() > 1)
892 setup_APIC_timer();
893 return;
894 }
895
896 /*
897 * If nmi_watchdog is set to IO_APIC, we need the
898 * PIT/HPET going. Otherwise register lapic as a dummy
899 * device.
900 */
901 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
902
903 /* Setup the lapic or request the broadcast */
904 setup_APIC_timer();
905 }
906
setup_secondary_APIC_clock(void)907 void setup_secondary_APIC_clock(void)
908 {
909 setup_APIC_timer();
910 }
911
912 /*
913 * The guts of the apic timer interrupt
914 */
local_apic_timer_interrupt(void)915 static void local_apic_timer_interrupt(void)
916 {
917 int cpu = smp_processor_id();
918 struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
919
920 /*
921 * Normally we should not be here till LAPIC has been initialized but
922 * in some cases like kdump, its possible that there is a pending LAPIC
923 * timer interrupt from previous kernel's context and is delivered in
924 * new kernel the moment interrupts are enabled.
925 *
926 * Interrupts are enabled early and LAPIC is setup much later, hence
927 * its possible that when we get here evt->event_handler is NULL.
928 * Check for event_handler being NULL and discard the interrupt as
929 * spurious.
930 */
931 if (!evt->event_handler) {
932 pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
933 /* Switch it off */
934 lapic_timer_shutdown(evt);
935 return;
936 }
937
938 /*
939 * the NMI deadlock-detector uses this.
940 */
941 inc_irq_stat(apic_timer_irqs);
942
943 evt->event_handler(evt);
944 }
945
946 /*
947 * Local APIC timer interrupt. This is the most natural way for doing
948 * local interrupts, but local timer interrupts can be emulated by
949 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
950 *
951 * [ if a single-CPU system runs an SMP kernel then we call the local
952 * interrupt as well. Thus we cannot inline the local irq ... ]
953 */
smp_apic_timer_interrupt(struct pt_regs * regs)954 __visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
955 {
956 struct pt_regs *old_regs = set_irq_regs(regs);
957
958 /*
959 * NOTE! We'd better ACK the irq immediately,
960 * because timer handling can be slow.
961 *
962 * update_process_times() expects us to have done irq_enter().
963 * Besides, if we don't timer interrupts ignore the global
964 * interrupt lock, which is the WrongThing (tm) to do.
965 */
966 entering_ack_irq();
967 local_apic_timer_interrupt();
968 exiting_irq();
969
970 set_irq_regs(old_regs);
971 }
972
smp_trace_apic_timer_interrupt(struct pt_regs * regs)973 __visible void __irq_entry smp_trace_apic_timer_interrupt(struct pt_regs *regs)
974 {
975 struct pt_regs *old_regs = set_irq_regs(regs);
976
977 /*
978 * NOTE! We'd better ACK the irq immediately,
979 * because timer handling can be slow.
980 *
981 * update_process_times() expects us to have done irq_enter().
982 * Besides, if we don't timer interrupts ignore the global
983 * interrupt lock, which is the WrongThing (tm) to do.
984 */
985 entering_ack_irq();
986 trace_local_timer_entry(LOCAL_TIMER_VECTOR);
987 local_apic_timer_interrupt();
988 trace_local_timer_exit(LOCAL_TIMER_VECTOR);
989 exiting_irq();
990
991 set_irq_regs(old_regs);
992 }
993
setup_profiling_timer(unsigned int multiplier)994 int setup_profiling_timer(unsigned int multiplier)
995 {
996 return -EINVAL;
997 }
998
999 /*
1000 * Local APIC start and shutdown
1001 */
1002
1003 /**
1004 * clear_local_APIC - shutdown the local APIC
1005 *
1006 * This is called, when a CPU is disabled and before rebooting, so the state of
1007 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1008 * leftovers during boot.
1009 */
clear_local_APIC(void)1010 void clear_local_APIC(void)
1011 {
1012 int maxlvt;
1013 u32 v;
1014
1015 /* APIC hasn't been mapped yet */
1016 if (!x2apic_mode && !apic_phys)
1017 return;
1018
1019 maxlvt = lapic_get_maxlvt();
1020 /*
1021 * Masking an LVT entry can trigger a local APIC error
1022 * if the vector is zero. Mask LVTERR first to prevent this.
1023 */
1024 if (maxlvt >= 3) {
1025 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1026 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1027 }
1028 /*
1029 * Careful: we have to set masks only first to deassert
1030 * any level-triggered sources.
1031 */
1032 v = apic_read(APIC_LVTT);
1033 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1034 v = apic_read(APIC_LVT0);
1035 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1036 v = apic_read(APIC_LVT1);
1037 apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1038 if (maxlvt >= 4) {
1039 v = apic_read(APIC_LVTPC);
1040 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1041 }
1042
1043 /* lets not touch this if we didn't frob it */
1044 #ifdef CONFIG_X86_THERMAL_VECTOR
1045 if (maxlvt >= 5) {
1046 v = apic_read(APIC_LVTTHMR);
1047 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1048 }
1049 #endif
1050 #ifdef CONFIG_X86_MCE_INTEL
1051 if (maxlvt >= 6) {
1052 v = apic_read(APIC_LVTCMCI);
1053 if (!(v & APIC_LVT_MASKED))
1054 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1055 }
1056 #endif
1057
1058 /*
1059 * Clean APIC state for other OSs:
1060 */
1061 apic_write(APIC_LVTT, APIC_LVT_MASKED);
1062 apic_write(APIC_LVT0, APIC_LVT_MASKED);
1063 apic_write(APIC_LVT1, APIC_LVT_MASKED);
1064 if (maxlvt >= 3)
1065 apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1066 if (maxlvt >= 4)
1067 apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1068
1069 /* Integrated APIC (!82489DX) ? */
1070 if (lapic_is_integrated()) {
1071 if (maxlvt > 3)
1072 /* Clear ESR due to Pentium errata 3AP and 11AP */
1073 apic_write(APIC_ESR, 0);
1074 apic_read(APIC_ESR);
1075 }
1076 }
1077
1078 /**
1079 * disable_local_APIC - clear and disable the local APIC
1080 */
disable_local_APIC(void)1081 void disable_local_APIC(void)
1082 {
1083 unsigned int value;
1084
1085 /* APIC hasn't been mapped yet */
1086 if (!x2apic_mode && !apic_phys)
1087 return;
1088
1089 clear_local_APIC();
1090
1091 /*
1092 * Disable APIC (implies clearing of registers
1093 * for 82489DX!).
1094 */
1095 value = apic_read(APIC_SPIV);
1096 value &= ~APIC_SPIV_APIC_ENABLED;
1097 apic_write(APIC_SPIV, value);
1098
1099 #ifdef CONFIG_X86_32
1100 /*
1101 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1102 * restore the disabled state.
1103 */
1104 if (enabled_via_apicbase) {
1105 unsigned int l, h;
1106
1107 rdmsr(MSR_IA32_APICBASE, l, h);
1108 l &= ~MSR_IA32_APICBASE_ENABLE;
1109 wrmsr(MSR_IA32_APICBASE, l, h);
1110 }
1111 #endif
1112 }
1113
1114 /*
1115 * If Linux enabled the LAPIC against the BIOS default disable it down before
1116 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1117 * not power-off. Additionally clear all LVT entries before disable_local_APIC
1118 * for the case where Linux didn't enable the LAPIC.
1119 */
lapic_shutdown(void)1120 void lapic_shutdown(void)
1121 {
1122 unsigned long flags;
1123
1124 if (!cpu_has_apic && !apic_from_smp_config())
1125 return;
1126
1127 local_irq_save(flags);
1128
1129 #ifdef CONFIG_X86_32
1130 if (!enabled_via_apicbase)
1131 clear_local_APIC();
1132 else
1133 #endif
1134 disable_local_APIC();
1135
1136
1137 local_irq_restore(flags);
1138 }
1139
1140 /**
1141 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1142 */
sync_Arb_IDs(void)1143 void __init sync_Arb_IDs(void)
1144 {
1145 /*
1146 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1147 * needed on AMD.
1148 */
1149 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1150 return;
1151
1152 /*
1153 * Wait for idle.
1154 */
1155 apic_wait_icr_idle();
1156
1157 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1158 apic_write(APIC_ICR, APIC_DEST_ALLINC |
1159 APIC_INT_LEVELTRIG | APIC_DM_INIT);
1160 }
1161
1162 /*
1163 * An initial setup of the virtual wire mode.
1164 */
init_bsp_APIC(void)1165 void __init init_bsp_APIC(void)
1166 {
1167 unsigned int value;
1168
1169 /*
1170 * Don't do the setup now if we have a SMP BIOS as the
1171 * through-I/O-APIC virtual wire mode might be active.
1172 */
1173 if (smp_found_config || !cpu_has_apic)
1174 return;
1175
1176 /*
1177 * Do not trust the local APIC being empty at bootup.
1178 */
1179 clear_local_APIC();
1180
1181 /*
1182 * Enable APIC.
1183 */
1184 value = apic_read(APIC_SPIV);
1185 value &= ~APIC_VECTOR_MASK;
1186 value |= APIC_SPIV_APIC_ENABLED;
1187
1188 #ifdef CONFIG_X86_32
1189 /* This bit is reserved on P4/Xeon and should be cleared */
1190 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1191 (boot_cpu_data.x86 == 15))
1192 value &= ~APIC_SPIV_FOCUS_DISABLED;
1193 else
1194 #endif
1195 value |= APIC_SPIV_FOCUS_DISABLED;
1196 value |= SPURIOUS_APIC_VECTOR;
1197 apic_write(APIC_SPIV, value);
1198
1199 /*
1200 * Set up the virtual wire mode.
1201 */
1202 apic_write(APIC_LVT0, APIC_DM_EXTINT);
1203 value = APIC_DM_NMI;
1204 if (!lapic_is_integrated()) /* 82489DX */
1205 value |= APIC_LVT_LEVEL_TRIGGER;
1206 apic_write(APIC_LVT1, value);
1207 }
1208
lapic_setup_esr(void)1209 static void lapic_setup_esr(void)
1210 {
1211 unsigned int oldvalue, value, maxlvt;
1212
1213 if (!lapic_is_integrated()) {
1214 pr_info("No ESR for 82489DX.\n");
1215 return;
1216 }
1217
1218 if (apic->disable_esr) {
1219 /*
1220 * Something untraceable is creating bad interrupts on
1221 * secondary quads ... for the moment, just leave the
1222 * ESR disabled - we can't do anything useful with the
1223 * errors anyway - mbligh
1224 */
1225 pr_info("Leaving ESR disabled.\n");
1226 return;
1227 }
1228
1229 maxlvt = lapic_get_maxlvt();
1230 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1231 apic_write(APIC_ESR, 0);
1232 oldvalue = apic_read(APIC_ESR);
1233
1234 /* enables sending errors */
1235 value = ERROR_APIC_VECTOR;
1236 apic_write(APIC_LVTERR, value);
1237
1238 /*
1239 * spec says clear errors after enabling vector.
1240 */
1241 if (maxlvt > 3)
1242 apic_write(APIC_ESR, 0);
1243 value = apic_read(APIC_ESR);
1244 if (value != oldvalue)
1245 apic_printk(APIC_VERBOSE, "ESR value before enabling "
1246 "vector: 0x%08x after: 0x%08x\n",
1247 oldvalue, value);
1248 }
1249
1250 /**
1251 * setup_local_APIC - setup the local APIC
1252 *
1253 * Used to setup local APIC while initializing BSP or bringin up APs.
1254 * Always called with preemption disabled.
1255 */
setup_local_APIC(void)1256 void setup_local_APIC(void)
1257 {
1258 int cpu = smp_processor_id();
1259 unsigned int value, queued;
1260 int i, j, acked = 0;
1261 unsigned long long tsc = 0, ntsc;
1262 long long max_loops = cpu_khz ? cpu_khz : 1000000;
1263
1264 if (cpu_has_tsc)
1265 tsc = rdtsc();
1266
1267 if (disable_apic) {
1268 disable_ioapic_support();
1269 return;
1270 }
1271
1272 /*
1273 * If this comes from kexec/kcrash the APIC might be enabled in
1274 * SPIV. Soft disable it before doing further initialization.
1275 */
1276 value = apic_read(APIC_SPIV);
1277 value &= ~APIC_SPIV_APIC_ENABLED;
1278 apic_write(APIC_SPIV, value);
1279
1280 #ifdef CONFIG_X86_32
1281 /* Pound the ESR really hard over the head with a big hammer - mbligh */
1282 if (lapic_is_integrated() && apic->disable_esr) {
1283 apic_write(APIC_ESR, 0);
1284 apic_write(APIC_ESR, 0);
1285 apic_write(APIC_ESR, 0);
1286 apic_write(APIC_ESR, 0);
1287 }
1288 #endif
1289 perf_events_lapic_init();
1290
1291 /*
1292 * Double-check whether this APIC is really registered.
1293 * This is meaningless in clustered apic mode, so we skip it.
1294 */
1295 BUG_ON(!apic->apic_id_registered());
1296
1297 /*
1298 * Intel recommends to set DFR, LDR and TPR before enabling
1299 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1300 * document number 292116). So here it goes...
1301 */
1302 apic->init_apic_ldr();
1303
1304 #ifdef CONFIG_X86_32
1305 if (apic->dest_logical) {
1306 int logical_apicid, ldr_apicid;
1307
1308 /*
1309 * APIC LDR is initialized. If logical_apicid mapping was
1310 * initialized during get_smp_config(), make sure it matches
1311 * the actual value.
1312 */
1313 logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
1314 ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
1315 if (logical_apicid != BAD_APICID)
1316 WARN_ON(logical_apicid != ldr_apicid);
1317 /* Always use the value from LDR. */
1318 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid;
1319 }
1320 #endif
1321
1322 /*
1323 * Set Task Priority to 'accept all'. We never change this
1324 * later on.
1325 */
1326 value = apic_read(APIC_TASKPRI);
1327 value &= ~APIC_TPRI_MASK;
1328 apic_write(APIC_TASKPRI, value);
1329
1330 /*
1331 * After a crash, we no longer service the interrupts and a pending
1332 * interrupt from previous kernel might still have ISR bit set.
1333 *
1334 * Most probably by now CPU has serviced that pending interrupt and
1335 * it might not have done the ack_APIC_irq() because it thought,
1336 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
1337 * does not clear the ISR bit and cpu thinks it has already serivced
1338 * the interrupt. Hence a vector might get locked. It was noticed
1339 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
1340 */
1341 do {
1342 queued = 0;
1343 for (i = APIC_ISR_NR - 1; i >= 0; i--)
1344 queued |= apic_read(APIC_IRR + i*0x10);
1345
1346 for (i = APIC_ISR_NR - 1; i >= 0; i--) {
1347 value = apic_read(APIC_ISR + i*0x10);
1348 for (j = 31; j >= 0; j--) {
1349 if (value & (1<<j)) {
1350 ack_APIC_irq();
1351 acked++;
1352 }
1353 }
1354 }
1355 if (acked > 256) {
1356 printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
1357 acked);
1358 break;
1359 }
1360 if (queued) {
1361 if (cpu_has_tsc && cpu_khz) {
1362 ntsc = rdtsc();
1363 max_loops = (cpu_khz << 10) - (ntsc - tsc);
1364 } else
1365 max_loops--;
1366 }
1367 } while (queued && max_loops > 0);
1368 WARN_ON(max_loops <= 0);
1369
1370 /*
1371 * Now that we are all set up, enable the APIC
1372 */
1373 value = apic_read(APIC_SPIV);
1374 value &= ~APIC_VECTOR_MASK;
1375 /*
1376 * Enable APIC
1377 */
1378 value |= APIC_SPIV_APIC_ENABLED;
1379
1380 #ifdef CONFIG_X86_32
1381 /*
1382 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1383 * certain networking cards. If high frequency interrupts are
1384 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1385 * entry is masked/unmasked at a high rate as well then sooner or
1386 * later IOAPIC line gets 'stuck', no more interrupts are received
1387 * from the device. If focus CPU is disabled then the hang goes
1388 * away, oh well :-(
1389 *
1390 * [ This bug can be reproduced easily with a level-triggered
1391 * PCI Ne2000 networking cards and PII/PIII processors, dual
1392 * BX chipset. ]
1393 */
1394 /*
1395 * Actually disabling the focus CPU check just makes the hang less
1396 * frequent as it makes the interrupt distributon model be more
1397 * like LRU than MRU (the short-term load is more even across CPUs).
1398 * See also the comment in end_level_ioapic_irq(). --macro
1399 */
1400
1401 /*
1402 * - enable focus processor (bit==0)
1403 * - 64bit mode always use processor focus
1404 * so no need to set it
1405 */
1406 value &= ~APIC_SPIV_FOCUS_DISABLED;
1407 #endif
1408
1409 /*
1410 * Set spurious IRQ vector
1411 */
1412 value |= SPURIOUS_APIC_VECTOR;
1413 apic_write(APIC_SPIV, value);
1414
1415 /*
1416 * Set up LVT0, LVT1:
1417 *
1418 * set up through-local-APIC on the BP's LINT0. This is not
1419 * strictly necessary in pure symmetric-IO mode, but sometimes
1420 * we delegate interrupts to the 8259A.
1421 */
1422 /*
1423 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1424 */
1425 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1426 if (!cpu && (pic_mode || !value || skip_ioapic_setup)) {
1427 value = APIC_DM_EXTINT;
1428 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1429 } else {
1430 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1431 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1432 }
1433 apic_write(APIC_LVT0, value);
1434
1435 /*
1436 * only the BP should see the LINT1 NMI signal, obviously.
1437 */
1438 if (!cpu)
1439 value = APIC_DM_NMI;
1440 else
1441 value = APIC_DM_NMI | APIC_LVT_MASKED;
1442 if (!lapic_is_integrated()) /* 82489DX */
1443 value |= APIC_LVT_LEVEL_TRIGGER;
1444 apic_write(APIC_LVT1, value);
1445
1446 #ifdef CONFIG_X86_MCE_INTEL
1447 /* Recheck CMCI information after local APIC is up on CPU #0 */
1448 if (!cpu)
1449 cmci_recheck();
1450 #endif
1451 }
1452
end_local_APIC_setup(void)1453 static void end_local_APIC_setup(void)
1454 {
1455 lapic_setup_esr();
1456
1457 #ifdef CONFIG_X86_32
1458 {
1459 unsigned int value;
1460 /* Disable the local apic timer */
1461 value = apic_read(APIC_LVTT);
1462 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1463 apic_write(APIC_LVTT, value);
1464 }
1465 #endif
1466
1467 apic_pm_activate();
1468 }
1469
1470 /*
1471 * APIC setup function for application processors. Called from smpboot.c
1472 */
apic_ap_setup(void)1473 void apic_ap_setup(void)
1474 {
1475 setup_local_APIC();
1476 end_local_APIC_setup();
1477 }
1478
1479 #ifdef CONFIG_X86_X2APIC
1480 int x2apic_mode;
1481
1482 enum {
1483 X2APIC_OFF,
1484 X2APIC_ON,
1485 X2APIC_DISABLED,
1486 };
1487 static int x2apic_state;
1488
__x2apic_disable(void)1489 static void __x2apic_disable(void)
1490 {
1491 u64 msr;
1492
1493 if (!cpu_has_apic)
1494 return;
1495
1496 rdmsrl(MSR_IA32_APICBASE, msr);
1497 if (!(msr & X2APIC_ENABLE))
1498 return;
1499 /* Disable xapic and x2apic first and then reenable xapic mode */
1500 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1501 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1502 printk_once(KERN_INFO "x2apic disabled\n");
1503 }
1504
__x2apic_enable(void)1505 static void __x2apic_enable(void)
1506 {
1507 u64 msr;
1508
1509 rdmsrl(MSR_IA32_APICBASE, msr);
1510 if (msr & X2APIC_ENABLE)
1511 return;
1512 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1513 printk_once(KERN_INFO "x2apic enabled\n");
1514 }
1515
setup_nox2apic(char * str)1516 static int __init setup_nox2apic(char *str)
1517 {
1518 if (x2apic_enabled()) {
1519 int apicid = native_apic_msr_read(APIC_ID);
1520
1521 if (apicid >= 255) {
1522 pr_warning("Apicid: %08x, cannot enforce nox2apic\n",
1523 apicid);
1524 return 0;
1525 }
1526 pr_warning("x2apic already enabled.\n");
1527 __x2apic_disable();
1528 }
1529 setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1530 x2apic_state = X2APIC_DISABLED;
1531 x2apic_mode = 0;
1532 return 0;
1533 }
1534 early_param("nox2apic", setup_nox2apic);
1535
1536 /* Called from cpu_init() to enable x2apic on (secondary) cpus */
x2apic_setup(void)1537 void x2apic_setup(void)
1538 {
1539 /*
1540 * If x2apic is not in ON state, disable it if already enabled
1541 * from BIOS.
1542 */
1543 if (x2apic_state != X2APIC_ON) {
1544 __x2apic_disable();
1545 return;
1546 }
1547 __x2apic_enable();
1548 }
1549
x2apic_disable(void)1550 static __init void x2apic_disable(void)
1551 {
1552 u32 x2apic_id, state = x2apic_state;
1553
1554 x2apic_mode = 0;
1555 x2apic_state = X2APIC_DISABLED;
1556
1557 if (state != X2APIC_ON)
1558 return;
1559
1560 x2apic_id = read_apic_id();
1561 if (x2apic_id >= 255)
1562 panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1563
1564 __x2apic_disable();
1565 register_lapic_address(mp_lapic_addr);
1566 }
1567
x2apic_enable(void)1568 static __init void x2apic_enable(void)
1569 {
1570 if (x2apic_state != X2APIC_OFF)
1571 return;
1572
1573 x2apic_mode = 1;
1574 x2apic_state = X2APIC_ON;
1575 __x2apic_enable();
1576 }
1577
try_to_enable_x2apic(int remap_mode)1578 static __init void try_to_enable_x2apic(int remap_mode)
1579 {
1580 if (x2apic_state == X2APIC_DISABLED)
1581 return;
1582
1583 if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1584 /* IR is required if there is APIC ID > 255 even when running
1585 * under KVM
1586 */
1587 if (max_physical_apicid > 255 ||
1588 !hypervisor_x2apic_available()) {
1589 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1590 x2apic_disable();
1591 return;
1592 }
1593
1594 /*
1595 * without IR all CPUs can be addressed by IOAPIC/MSI
1596 * only in physical mode
1597 */
1598 x2apic_phys = 1;
1599 }
1600 x2apic_enable();
1601 }
1602
check_x2apic(void)1603 void __init check_x2apic(void)
1604 {
1605 if (x2apic_enabled()) {
1606 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1607 x2apic_mode = 1;
1608 x2apic_state = X2APIC_ON;
1609 } else if (!cpu_has_x2apic) {
1610 x2apic_state = X2APIC_DISABLED;
1611 }
1612 }
1613 #else /* CONFIG_X86_X2APIC */
validate_x2apic(void)1614 static int __init validate_x2apic(void)
1615 {
1616 if (!apic_is_x2apic_enabled())
1617 return 0;
1618 /*
1619 * Checkme: Can we simply turn off x2apic here instead of panic?
1620 */
1621 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n");
1622 }
1623 early_initcall(validate_x2apic);
1624
try_to_enable_x2apic(int remap_mode)1625 static inline void try_to_enable_x2apic(int remap_mode) { }
__x2apic_enable(void)1626 static inline void __x2apic_enable(void) { }
1627 #endif /* !CONFIG_X86_X2APIC */
1628
try_to_enable_IR(void)1629 static int __init try_to_enable_IR(void)
1630 {
1631 #ifdef CONFIG_X86_IO_APIC
1632 if (!x2apic_enabled() && skip_ioapic_setup) {
1633 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1634 return -1;
1635 }
1636 #endif
1637 return irq_remapping_enable();
1638 }
1639
enable_IR_x2apic(void)1640 void __init enable_IR_x2apic(void)
1641 {
1642 unsigned long flags;
1643 int ret, ir_stat;
1644
1645 if (skip_ioapic_setup)
1646 return;
1647
1648 ir_stat = irq_remapping_prepare();
1649 if (ir_stat < 0 && !x2apic_supported())
1650 return;
1651
1652 ret = save_ioapic_entries();
1653 if (ret) {
1654 pr_info("Saving IO-APIC state failed: %d\n", ret);
1655 return;
1656 }
1657
1658 local_irq_save(flags);
1659 legacy_pic->mask_all();
1660 mask_ioapic_entries();
1661
1662 /* If irq_remapping_prepare() succeded, try to enable it */
1663 if (ir_stat >= 0)
1664 ir_stat = try_to_enable_IR();
1665 /* ir_stat contains the remap mode or an error code */
1666 try_to_enable_x2apic(ir_stat);
1667
1668 if (ir_stat < 0)
1669 restore_ioapic_entries();
1670 legacy_pic->restore_mask();
1671 local_irq_restore(flags);
1672 }
1673
1674 #ifdef CONFIG_X86_64
1675 /*
1676 * Detect and enable local APICs on non-SMP boards.
1677 * Original code written by Keir Fraser.
1678 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1679 * not correctly set up (usually the APIC timer won't work etc.)
1680 */
detect_init_APIC(void)1681 static int __init detect_init_APIC(void)
1682 {
1683 if (!cpu_has_apic) {
1684 pr_info("No local APIC present\n");
1685 return -1;
1686 }
1687
1688 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1689 return 0;
1690 }
1691 #else
1692
apic_verify(void)1693 static int __init apic_verify(void)
1694 {
1695 u32 features, h, l;
1696
1697 /*
1698 * The APIC feature bit should now be enabled
1699 * in `cpuid'
1700 */
1701 features = cpuid_edx(1);
1702 if (!(features & (1 << X86_FEATURE_APIC))) {
1703 pr_warning("Could not enable APIC!\n");
1704 return -1;
1705 }
1706 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1707 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1708
1709 /* The BIOS may have set up the APIC at some other address */
1710 if (boot_cpu_data.x86 >= 6) {
1711 rdmsr(MSR_IA32_APICBASE, l, h);
1712 if (l & MSR_IA32_APICBASE_ENABLE)
1713 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1714 }
1715
1716 pr_info("Found and enabled local APIC!\n");
1717 return 0;
1718 }
1719
apic_force_enable(unsigned long addr)1720 int __init apic_force_enable(unsigned long addr)
1721 {
1722 u32 h, l;
1723
1724 if (disable_apic)
1725 return -1;
1726
1727 /*
1728 * Some BIOSes disable the local APIC in the APIC_BASE
1729 * MSR. This can only be done in software for Intel P6 or later
1730 * and AMD K7 (Model > 1) or later.
1731 */
1732 if (boot_cpu_data.x86 >= 6) {
1733 rdmsr(MSR_IA32_APICBASE, l, h);
1734 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1735 pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1736 l &= ~MSR_IA32_APICBASE_BASE;
1737 l |= MSR_IA32_APICBASE_ENABLE | addr;
1738 wrmsr(MSR_IA32_APICBASE, l, h);
1739 enabled_via_apicbase = 1;
1740 }
1741 }
1742 return apic_verify();
1743 }
1744
1745 /*
1746 * Detect and initialize APIC
1747 */
detect_init_APIC(void)1748 static int __init detect_init_APIC(void)
1749 {
1750 /* Disabled by kernel option? */
1751 if (disable_apic)
1752 return -1;
1753
1754 switch (boot_cpu_data.x86_vendor) {
1755 case X86_VENDOR_AMD:
1756 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
1757 (boot_cpu_data.x86 >= 15))
1758 break;
1759 goto no_apic;
1760 case X86_VENDOR_INTEL:
1761 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
1762 (boot_cpu_data.x86 == 5 && cpu_has_apic))
1763 break;
1764 goto no_apic;
1765 default:
1766 goto no_apic;
1767 }
1768
1769 if (!cpu_has_apic) {
1770 /*
1771 * Over-ride BIOS and try to enable the local APIC only if
1772 * "lapic" specified.
1773 */
1774 if (!force_enable_local_apic) {
1775 pr_info("Local APIC disabled by BIOS -- "
1776 "you can enable it with \"lapic\"\n");
1777 return -1;
1778 }
1779 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
1780 return -1;
1781 } else {
1782 if (apic_verify())
1783 return -1;
1784 }
1785
1786 apic_pm_activate();
1787
1788 return 0;
1789
1790 no_apic:
1791 pr_info("No local APIC present or hardware disabled\n");
1792 return -1;
1793 }
1794 #endif
1795
1796 /**
1797 * init_apic_mappings - initialize APIC mappings
1798 */
init_apic_mappings(void)1799 void __init init_apic_mappings(void)
1800 {
1801 unsigned int new_apicid;
1802
1803 if (x2apic_mode) {
1804 boot_cpu_physical_apicid = read_apic_id();
1805 return;
1806 }
1807
1808 /* If no local APIC can be found return early */
1809 if (!smp_found_config && detect_init_APIC()) {
1810 /* lets NOP'ify apic operations */
1811 pr_info("APIC: disable apic facility\n");
1812 apic_disable();
1813 } else {
1814 apic_phys = mp_lapic_addr;
1815
1816 /*
1817 * acpi lapic path already maps that address in
1818 * acpi_register_lapic_address()
1819 */
1820 if (!acpi_lapic && !smp_found_config)
1821 register_lapic_address(apic_phys);
1822 }
1823
1824 /*
1825 * Fetch the APIC ID of the BSP in case we have a
1826 * default configuration (or the MP table is broken).
1827 */
1828 new_apicid = read_apic_id();
1829 if (boot_cpu_physical_apicid != new_apicid) {
1830 boot_cpu_physical_apicid = new_apicid;
1831 /*
1832 * yeah -- we lie about apic_version
1833 * in case if apic was disabled via boot option
1834 * but it's not a problem for SMP compiled kernel
1835 * since smp_sanity_check is prepared for such a case
1836 * and disable smp mode
1837 */
1838 apic_version[new_apicid] =
1839 GET_APIC_VERSION(apic_read(APIC_LVR));
1840 }
1841 }
1842
register_lapic_address(unsigned long address)1843 void __init register_lapic_address(unsigned long address)
1844 {
1845 mp_lapic_addr = address;
1846
1847 if (!x2apic_mode) {
1848 set_fixmap_nocache(FIX_APIC_BASE, address);
1849 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1850 APIC_BASE, mp_lapic_addr);
1851 }
1852 if (boot_cpu_physical_apicid == -1U) {
1853 boot_cpu_physical_apicid = read_apic_id();
1854 apic_version[boot_cpu_physical_apicid] =
1855 GET_APIC_VERSION(apic_read(APIC_LVR));
1856 }
1857 }
1858
1859 int apic_version[MAX_LOCAL_APIC];
1860
1861 /*
1862 * Local APIC interrupts
1863 */
1864
1865 /*
1866 * This interrupt should _never_ happen with our APIC/SMP architecture
1867 */
__smp_spurious_interrupt(u8 vector)1868 static void __smp_spurious_interrupt(u8 vector)
1869 {
1870 u32 v;
1871
1872 /*
1873 * Check if this really is a spurious interrupt and ACK it
1874 * if it is a vectored one. Just in case...
1875 * Spurious interrupts should not be ACKed.
1876 */
1877 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
1878 if (v & (1 << (vector & 0x1f)))
1879 ack_APIC_irq();
1880
1881 inc_irq_stat(irq_spurious_count);
1882
1883 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1884 pr_info("spurious APIC interrupt through vector %02x on CPU#%d, "
1885 "should never happen.\n", vector, smp_processor_id());
1886 }
1887
smp_spurious_interrupt(struct pt_regs * regs)1888 __visible void smp_spurious_interrupt(struct pt_regs *regs)
1889 {
1890 entering_irq();
1891 __smp_spurious_interrupt(~regs->orig_ax);
1892 exiting_irq();
1893 }
1894
smp_trace_spurious_interrupt(struct pt_regs * regs)1895 __visible void smp_trace_spurious_interrupt(struct pt_regs *regs)
1896 {
1897 u8 vector = ~regs->orig_ax;
1898
1899 entering_irq();
1900 trace_spurious_apic_entry(vector);
1901 __smp_spurious_interrupt(vector);
1902 trace_spurious_apic_exit(vector);
1903 exiting_irq();
1904 }
1905
1906 /*
1907 * This interrupt should never happen with our APIC/SMP architecture
1908 */
__smp_error_interrupt(struct pt_regs * regs)1909 static void __smp_error_interrupt(struct pt_regs *regs)
1910 {
1911 u32 v;
1912 u32 i = 0;
1913 static const char * const error_interrupt_reason[] = {
1914 "Send CS error", /* APIC Error Bit 0 */
1915 "Receive CS error", /* APIC Error Bit 1 */
1916 "Send accept error", /* APIC Error Bit 2 */
1917 "Receive accept error", /* APIC Error Bit 3 */
1918 "Redirectable IPI", /* APIC Error Bit 4 */
1919 "Send illegal vector", /* APIC Error Bit 5 */
1920 "Received illegal vector", /* APIC Error Bit 6 */
1921 "Illegal register address", /* APIC Error Bit 7 */
1922 };
1923
1924 /* First tickle the hardware, only then report what went on. -- REW */
1925 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
1926 apic_write(APIC_ESR, 0);
1927 v = apic_read(APIC_ESR);
1928 ack_APIC_irq();
1929 atomic_inc(&irq_err_count);
1930
1931 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
1932 smp_processor_id(), v);
1933
1934 v &= 0xff;
1935 while (v) {
1936 if (v & 0x1)
1937 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
1938 i++;
1939 v >>= 1;
1940 }
1941
1942 apic_printk(APIC_DEBUG, KERN_CONT "\n");
1943
1944 }
1945
smp_error_interrupt(struct pt_regs * regs)1946 __visible void smp_error_interrupt(struct pt_regs *regs)
1947 {
1948 entering_irq();
1949 __smp_error_interrupt(regs);
1950 exiting_irq();
1951 }
1952
smp_trace_error_interrupt(struct pt_regs * regs)1953 __visible void smp_trace_error_interrupt(struct pt_regs *regs)
1954 {
1955 entering_irq();
1956 trace_error_apic_entry(ERROR_APIC_VECTOR);
1957 __smp_error_interrupt(regs);
1958 trace_error_apic_exit(ERROR_APIC_VECTOR);
1959 exiting_irq();
1960 }
1961
1962 /**
1963 * connect_bsp_APIC - attach the APIC to the interrupt system
1964 */
connect_bsp_APIC(void)1965 static void __init connect_bsp_APIC(void)
1966 {
1967 #ifdef CONFIG_X86_32
1968 if (pic_mode) {
1969 /*
1970 * Do not trust the local APIC being empty at bootup.
1971 */
1972 clear_local_APIC();
1973 /*
1974 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
1975 * local APIC to INT and NMI lines.
1976 */
1977 apic_printk(APIC_VERBOSE, "leaving PIC mode, "
1978 "enabling APIC mode.\n");
1979 imcr_pic_to_apic();
1980 }
1981 #endif
1982 }
1983
1984 /**
1985 * disconnect_bsp_APIC - detach the APIC from the interrupt system
1986 * @virt_wire_setup: indicates, whether virtual wire mode is selected
1987 *
1988 * Virtual wire mode is necessary to deliver legacy interrupts even when the
1989 * APIC is disabled.
1990 */
disconnect_bsp_APIC(int virt_wire_setup)1991 void disconnect_bsp_APIC(int virt_wire_setup)
1992 {
1993 unsigned int value;
1994
1995 #ifdef CONFIG_X86_32
1996 if (pic_mode) {
1997 /*
1998 * Put the board back into PIC mode (has an effect only on
1999 * certain older boards). Note that APIC interrupts, including
2000 * IPIs, won't work beyond this point! The only exception are
2001 * INIT IPIs.
2002 */
2003 apic_printk(APIC_VERBOSE, "disabling APIC mode, "
2004 "entering PIC mode.\n");
2005 imcr_apic_to_pic();
2006 return;
2007 }
2008 #endif
2009
2010 /* Go back to Virtual Wire compatibility mode */
2011
2012 /* For the spurious interrupt use vector F, and enable it */
2013 value = apic_read(APIC_SPIV);
2014 value &= ~APIC_VECTOR_MASK;
2015 value |= APIC_SPIV_APIC_ENABLED;
2016 value |= 0xf;
2017 apic_write(APIC_SPIV, value);
2018
2019 if (!virt_wire_setup) {
2020 /*
2021 * For LVT0 make it edge triggered, active high,
2022 * external and enabled
2023 */
2024 value = apic_read(APIC_LVT0);
2025 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2026 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2027 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2028 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2029 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2030 apic_write(APIC_LVT0, value);
2031 } else {
2032 /* Disable LVT0 */
2033 apic_write(APIC_LVT0, APIC_LVT_MASKED);
2034 }
2035
2036 /*
2037 * For LVT1 make it edge triggered, active high,
2038 * nmi and enabled
2039 */
2040 value = apic_read(APIC_LVT1);
2041 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2042 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2043 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2044 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2045 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2046 apic_write(APIC_LVT1, value);
2047 }
2048
generic_processor_info(int apicid,int version)2049 int generic_processor_info(int apicid, int version)
2050 {
2051 int cpu, max = nr_cpu_ids;
2052 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
2053 phys_cpu_present_map);
2054
2055 /*
2056 * boot_cpu_physical_apicid is designed to have the apicid
2057 * returned by read_apic_id(), i.e, the apicid of the
2058 * currently booting-up processor. However, on some platforms,
2059 * it is temporarily modified by the apicid reported as BSP
2060 * through MP table. Concretely:
2061 *
2062 * - arch/x86/kernel/mpparse.c: MP_processor_info()
2063 * - arch/x86/mm/amdtopology.c: amd_numa_init()
2064 *
2065 * This function is executed with the modified
2066 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
2067 * parameter doesn't work to disable APs on kdump 2nd kernel.
2068 *
2069 * Since fixing handling of boot_cpu_physical_apicid requires
2070 * another discussion and tests on each platform, we leave it
2071 * for now and here we use read_apic_id() directly in this
2072 * function, generic_processor_info().
2073 */
2074 if (disabled_cpu_apicid != BAD_APICID &&
2075 disabled_cpu_apicid != read_apic_id() &&
2076 disabled_cpu_apicid == apicid) {
2077 int thiscpu = num_processors + disabled_cpus;
2078
2079 pr_warning("APIC: Disabling requested cpu."
2080 " Processor %d/0x%x ignored.\n",
2081 thiscpu, apicid);
2082
2083 disabled_cpus++;
2084 return -ENODEV;
2085 }
2086
2087 /*
2088 * If boot cpu has not been detected yet, then only allow upto
2089 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2090 */
2091 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
2092 apicid != boot_cpu_physical_apicid) {
2093 int thiscpu = max + disabled_cpus - 1;
2094
2095 pr_warning(
2096 "ACPI: NR_CPUS/possible_cpus limit of %i almost"
2097 " reached. Keeping one slot for boot cpu."
2098 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2099
2100 disabled_cpus++;
2101 return -ENODEV;
2102 }
2103
2104 if (num_processors >= nr_cpu_ids) {
2105 int thiscpu = max + disabled_cpus;
2106
2107 pr_warning(
2108 "ACPI: NR_CPUS/possible_cpus limit of %i reached."
2109 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2110
2111 disabled_cpus++;
2112 return -EINVAL;
2113 }
2114
2115 num_processors++;
2116 if (apicid == boot_cpu_physical_apicid) {
2117 /*
2118 * x86_bios_cpu_apicid is required to have processors listed
2119 * in same order as logical cpu numbers. Hence the first
2120 * entry is BSP, and so on.
2121 * boot_cpu_init() already hold bit 0 in cpu_present_mask
2122 * for BSP.
2123 */
2124 cpu = 0;
2125 } else
2126 cpu = cpumask_next_zero(-1, cpu_present_mask);
2127
2128 /*
2129 * Validate version
2130 */
2131 if (version == 0x0) {
2132 pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2133 cpu, apicid);
2134 version = 0x10;
2135 }
2136 apic_version[apicid] = version;
2137
2138 if (version != apic_version[boot_cpu_physical_apicid]) {
2139 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2140 apic_version[boot_cpu_physical_apicid], cpu, version);
2141 }
2142
2143 physid_set(apicid, phys_cpu_present_map);
2144 if (apicid > max_physical_apicid)
2145 max_physical_apicid = apicid;
2146
2147 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2148 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2149 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
2150 #endif
2151 #ifdef CONFIG_X86_32
2152 early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
2153 apic->x86_32_early_logical_apicid(cpu);
2154 #endif
2155 set_cpu_possible(cpu, true);
2156 set_cpu_present(cpu, true);
2157
2158 return cpu;
2159 }
2160
hard_smp_processor_id(void)2161 int hard_smp_processor_id(void)
2162 {
2163 return read_apic_id();
2164 }
2165
default_init_apic_ldr(void)2166 void default_init_apic_ldr(void)
2167 {
2168 unsigned long val;
2169
2170 apic_write(APIC_DFR, APIC_DFR_VALUE);
2171 val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
2172 val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
2173 apic_write(APIC_LDR, val);
2174 }
2175
default_cpu_mask_to_apicid_and(const struct cpumask * cpumask,const struct cpumask * andmask,unsigned int * apicid)2176 int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
2177 const struct cpumask *andmask,
2178 unsigned int *apicid)
2179 {
2180 unsigned int cpu;
2181
2182 for_each_cpu_and(cpu, cpumask, andmask) {
2183 if (cpumask_test_cpu(cpu, cpu_online_mask))
2184 break;
2185 }
2186
2187 if (likely(cpu < nr_cpu_ids)) {
2188 *apicid = per_cpu(x86_cpu_to_apicid, cpu);
2189 return 0;
2190 }
2191
2192 return -EINVAL;
2193 }
2194
2195 /*
2196 * Override the generic EOI implementation with an optimized version.
2197 * Only called during early boot when only one CPU is active and with
2198 * interrupts disabled, so we know this does not race with actual APIC driver
2199 * use.
2200 */
apic_set_eoi_write(void (* eoi_write)(u32 reg,u32 v))2201 void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v))
2202 {
2203 struct apic **drv;
2204
2205 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
2206 /* Should happen once for each apic */
2207 WARN_ON((*drv)->eoi_write == eoi_write);
2208 (*drv)->eoi_write = eoi_write;
2209 }
2210 }
2211
apic_bsp_up_setup(void)2212 static void __init apic_bsp_up_setup(void)
2213 {
2214 #ifdef CONFIG_X86_64
2215 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
2216 #else
2217 /*
2218 * Hack: In case of kdump, after a crash, kernel might be booting
2219 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
2220 * might be zero if read from MP tables. Get it from LAPIC.
2221 */
2222 # ifdef CONFIG_CRASH_DUMP
2223 boot_cpu_physical_apicid = read_apic_id();
2224 # endif
2225 #endif
2226 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
2227 }
2228
2229 /**
2230 * apic_bsp_setup - Setup function for local apic and io-apic
2231 * @upmode: Force UP mode (for APIC_init_uniprocessor)
2232 *
2233 * Returns:
2234 * apic_id of BSP APIC
2235 */
apic_bsp_setup(bool upmode)2236 int __init apic_bsp_setup(bool upmode)
2237 {
2238 int id;
2239
2240 connect_bsp_APIC();
2241 if (upmode)
2242 apic_bsp_up_setup();
2243 setup_local_APIC();
2244
2245 if (x2apic_mode)
2246 id = apic_read(APIC_LDR);
2247 else
2248 id = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
2249
2250 enable_IO_APIC();
2251 end_local_APIC_setup();
2252 irq_remap_enable_fault_handling();
2253 setup_IO_APIC();
2254 /* Setup local timer */
2255 x86_init.timers.setup_percpu_clockev();
2256 return id;
2257 }
2258
2259 /*
2260 * This initializes the IO-APIC and APIC hardware if this is
2261 * a UP kernel.
2262 */
APIC_init_uniprocessor(void)2263 int __init APIC_init_uniprocessor(void)
2264 {
2265 if (disable_apic) {
2266 pr_info("Apic disabled\n");
2267 return -1;
2268 }
2269 #ifdef CONFIG_X86_64
2270 if (!cpu_has_apic) {
2271 disable_apic = 1;
2272 pr_info("Apic disabled by BIOS\n");
2273 return -1;
2274 }
2275 #else
2276 if (!smp_found_config && !cpu_has_apic)
2277 return -1;
2278
2279 /*
2280 * Complain if the BIOS pretends there is one.
2281 */
2282 if (!cpu_has_apic &&
2283 APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
2284 pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
2285 boot_cpu_physical_apicid);
2286 return -1;
2287 }
2288 #endif
2289
2290 if (!smp_found_config)
2291 disable_ioapic_support();
2292
2293 default_setup_apic_routing();
2294 apic_bsp_setup(true);
2295 return 0;
2296 }
2297
2298 #ifdef CONFIG_UP_LATE_INIT
up_late_init(void)2299 void __init up_late_init(void)
2300 {
2301 APIC_init_uniprocessor();
2302 }
2303 #endif
2304
2305 /*
2306 * Power management
2307 */
2308 #ifdef CONFIG_PM
2309
2310 static struct {
2311 /*
2312 * 'active' is true if the local APIC was enabled by us and
2313 * not the BIOS; this signifies that we are also responsible
2314 * for disabling it before entering apm/acpi suspend
2315 */
2316 int active;
2317 /* r/w apic fields */
2318 unsigned int apic_id;
2319 unsigned int apic_taskpri;
2320 unsigned int apic_ldr;
2321 unsigned int apic_dfr;
2322 unsigned int apic_spiv;
2323 unsigned int apic_lvtt;
2324 unsigned int apic_lvtpc;
2325 unsigned int apic_lvt0;
2326 unsigned int apic_lvt1;
2327 unsigned int apic_lvterr;
2328 unsigned int apic_tmict;
2329 unsigned int apic_tdcr;
2330 unsigned int apic_thmr;
2331 } apic_pm_state;
2332
lapic_suspend(void)2333 static int lapic_suspend(void)
2334 {
2335 unsigned long flags;
2336 int maxlvt;
2337
2338 if (!apic_pm_state.active)
2339 return 0;
2340
2341 maxlvt = lapic_get_maxlvt();
2342
2343 apic_pm_state.apic_id = apic_read(APIC_ID);
2344 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2345 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2346 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2347 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2348 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2349 if (maxlvt >= 4)
2350 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2351 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2352 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2353 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2354 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2355 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2356 #ifdef CONFIG_X86_THERMAL_VECTOR
2357 if (maxlvt >= 5)
2358 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2359 #endif
2360
2361 local_irq_save(flags);
2362 disable_local_APIC();
2363
2364 irq_remapping_disable();
2365
2366 local_irq_restore(flags);
2367 return 0;
2368 }
2369
lapic_resume(void)2370 static void lapic_resume(void)
2371 {
2372 unsigned int l, h;
2373 unsigned long flags;
2374 int maxlvt;
2375
2376 if (!apic_pm_state.active)
2377 return;
2378
2379 local_irq_save(flags);
2380
2381 /*
2382 * IO-APIC and PIC have their own resume routines.
2383 * We just mask them here to make sure the interrupt
2384 * subsystem is completely quiet while we enable x2apic
2385 * and interrupt-remapping.
2386 */
2387 mask_ioapic_entries();
2388 legacy_pic->mask_all();
2389
2390 if (x2apic_mode) {
2391 __x2apic_enable();
2392 } else {
2393 /*
2394 * Make sure the APICBASE points to the right address
2395 *
2396 * FIXME! This will be wrong if we ever support suspend on
2397 * SMP! We'll need to do this as part of the CPU restore!
2398 */
2399 if (boot_cpu_data.x86 >= 6) {
2400 rdmsr(MSR_IA32_APICBASE, l, h);
2401 l &= ~MSR_IA32_APICBASE_BASE;
2402 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2403 wrmsr(MSR_IA32_APICBASE, l, h);
2404 }
2405 }
2406
2407 maxlvt = lapic_get_maxlvt();
2408 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2409 apic_write(APIC_ID, apic_pm_state.apic_id);
2410 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2411 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2412 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2413 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2414 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2415 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2416 #if defined(CONFIG_X86_MCE_INTEL)
2417 if (maxlvt >= 5)
2418 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2419 #endif
2420 if (maxlvt >= 4)
2421 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2422 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2423 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2424 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2425 apic_write(APIC_ESR, 0);
2426 apic_read(APIC_ESR);
2427 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2428 apic_write(APIC_ESR, 0);
2429 apic_read(APIC_ESR);
2430
2431 irq_remapping_reenable(x2apic_mode);
2432
2433 local_irq_restore(flags);
2434 }
2435
2436 /*
2437 * This device has no shutdown method - fully functioning local APICs
2438 * are needed on every CPU up until machine_halt/restart/poweroff.
2439 */
2440
2441 static struct syscore_ops lapic_syscore_ops = {
2442 .resume = lapic_resume,
2443 .suspend = lapic_suspend,
2444 };
2445
apic_pm_activate(void)2446 static void apic_pm_activate(void)
2447 {
2448 apic_pm_state.active = 1;
2449 }
2450
init_lapic_sysfs(void)2451 static int __init init_lapic_sysfs(void)
2452 {
2453 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2454 if (cpu_has_apic)
2455 register_syscore_ops(&lapic_syscore_ops);
2456
2457 return 0;
2458 }
2459
2460 /* local apic needs to resume before other devices access its registers. */
2461 core_initcall(init_lapic_sysfs);
2462
2463 #else /* CONFIG_PM */
2464
apic_pm_activate(void)2465 static void apic_pm_activate(void) { }
2466
2467 #endif /* CONFIG_PM */
2468
2469 #ifdef CONFIG_X86_64
2470
2471 static int multi_checked;
2472 static int multi;
2473
set_multi(const struct dmi_system_id * d)2474 static int set_multi(const struct dmi_system_id *d)
2475 {
2476 if (multi)
2477 return 0;
2478 pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2479 multi = 1;
2480 return 0;
2481 }
2482
2483 static const struct dmi_system_id multi_dmi_table[] = {
2484 {
2485 .callback = set_multi,
2486 .ident = "IBM System Summit2",
2487 .matches = {
2488 DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2489 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2490 },
2491 },
2492 {}
2493 };
2494
dmi_check_multi(void)2495 static void dmi_check_multi(void)
2496 {
2497 if (multi_checked)
2498 return;
2499
2500 dmi_check_system(multi_dmi_table);
2501 multi_checked = 1;
2502 }
2503
2504 /*
2505 * apic_is_clustered_box() -- Check if we can expect good TSC
2506 *
2507 * Thus far, the major user of this is IBM's Summit2 series:
2508 * Clustered boxes may have unsynced TSC problems if they are
2509 * multi-chassis.
2510 * Use DMI to check them
2511 */
apic_is_clustered_box(void)2512 int apic_is_clustered_box(void)
2513 {
2514 dmi_check_multi();
2515 return multi;
2516 }
2517 #endif
2518
2519 /*
2520 * APIC command line parameters
2521 */
setup_disableapic(char * arg)2522 static int __init setup_disableapic(char *arg)
2523 {
2524 disable_apic = 1;
2525 setup_clear_cpu_cap(X86_FEATURE_APIC);
2526 return 0;
2527 }
2528 early_param("disableapic", setup_disableapic);
2529
2530 /* same as disableapic, for compatibility */
setup_nolapic(char * arg)2531 static int __init setup_nolapic(char *arg)
2532 {
2533 return setup_disableapic(arg);
2534 }
2535 early_param("nolapic", setup_nolapic);
2536
parse_lapic_timer_c2_ok(char * arg)2537 static int __init parse_lapic_timer_c2_ok(char *arg)
2538 {
2539 local_apic_timer_c2_ok = 1;
2540 return 0;
2541 }
2542 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2543
parse_disable_apic_timer(char * arg)2544 static int __init parse_disable_apic_timer(char *arg)
2545 {
2546 disable_apic_timer = 1;
2547 return 0;
2548 }
2549 early_param("noapictimer", parse_disable_apic_timer);
2550
parse_nolapic_timer(char * arg)2551 static int __init parse_nolapic_timer(char *arg)
2552 {
2553 disable_apic_timer = 1;
2554 return 0;
2555 }
2556 early_param("nolapic_timer", parse_nolapic_timer);
2557
apic_set_verbosity(char * arg)2558 static int __init apic_set_verbosity(char *arg)
2559 {
2560 if (!arg) {
2561 #ifdef CONFIG_X86_64
2562 skip_ioapic_setup = 0;
2563 return 0;
2564 #endif
2565 return -EINVAL;
2566 }
2567
2568 if (strcmp("debug", arg) == 0)
2569 apic_verbosity = APIC_DEBUG;
2570 else if (strcmp("verbose", arg) == 0)
2571 apic_verbosity = APIC_VERBOSE;
2572 else {
2573 pr_warning("APIC Verbosity level %s not recognised"
2574 " use apic=verbose or apic=debug\n", arg);
2575 return -EINVAL;
2576 }
2577
2578 return 0;
2579 }
2580 early_param("apic", apic_set_verbosity);
2581
lapic_insert_resource(void)2582 static int __init lapic_insert_resource(void)
2583 {
2584 if (!apic_phys)
2585 return -1;
2586
2587 /* Put local APIC into the resource map. */
2588 lapic_resource.start = apic_phys;
2589 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2590 insert_resource(&iomem_resource, &lapic_resource);
2591
2592 return 0;
2593 }
2594
2595 /*
2596 * need call insert after e820_reserve_resources()
2597 * that is using request_resource
2598 */
2599 late_initcall(lapic_insert_resource);
2600
apic_set_disabled_cpu_apicid(char * arg)2601 static int __init apic_set_disabled_cpu_apicid(char *arg)
2602 {
2603 if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2604 return -EINVAL;
2605
2606 return 0;
2607 }
2608 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);
2609