• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  linux/arch/arm/kernel/smp.c
3  *
4  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 #include <linux/module.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
19 #include <linux/mm.h>
20 #include <linux/err.h>
21 #include <linux/cpu.h>
22 #include <linux/seq_file.h>
23 #include <linux/irq.h>
24 #include <linux/nmi.h>
25 #include <linux/percpu.h>
26 #include <linux/clockchips.h>
27 #include <linux/completion.h>
28 #include <linux/cpufreq.h>
29 #include <linux/irq_work.h>
30 #include <linux/slab.h>
31 
32 #include <linux/atomic.h>
33 #include <asm/bugs.h>
34 #include <asm/smp.h>
35 #include <asm/cacheflush.h>
36 #include <asm/cpu.h>
37 #include <asm/cputype.h>
38 #include <asm/exception.h>
39 #include <asm/idmap.h>
40 #include <asm/topology.h>
41 #include <asm/mmu_context.h>
42 #include <asm/pgtable.h>
43 #include <asm/pgalloc.h>
44 #include <asm/procinfo.h>
45 #include <asm/processor.h>
46 #include <asm/sections.h>
47 #include <asm/tlbflush.h>
48 #include <asm/ptrace.h>
49 #include <asm/smp_plat.h>
50 #include <asm/virt.h>
51 #include <asm/mach/arch.h>
52 #include <asm/mpu.h>
53 
54 #define CREATE_TRACE_POINTS
55 #include <trace/events/ipi.h>
56 
57 /*
58  * as from 2.5, kernels no longer have an init_tasks structure
59  * so we need some other way of telling a new secondary core
60  * where to place its SVC stack
61  */
62 struct secondary_data secondary_data;
63 
64 /*
65  * control for which core is the next to come out of the secondary
66  * boot "holding pen"
67  */
68 volatile int pen_release = -1;
69 
70 enum ipi_msg_type {
71 	IPI_WAKEUP,
72 	IPI_TIMER,
73 	IPI_RESCHEDULE,
74 	IPI_CALL_FUNC,
75 	IPI_CALL_FUNC_SINGLE,
76 	IPI_CPU_STOP,
77 	IPI_IRQ_WORK,
78 	IPI_COMPLETION,
79 	IPI_CPU_BACKTRACE = 15,
80 };
81 
82 static DECLARE_COMPLETION(cpu_running);
83 
84 static struct smp_operations smp_ops;
85 
smp_set_ops(const struct smp_operations * ops)86 void __init smp_set_ops(const struct smp_operations *ops)
87 {
88 	if (ops)
89 		smp_ops = *ops;
90 };
91 
get_arch_pgd(pgd_t * pgd)92 static unsigned long get_arch_pgd(pgd_t *pgd)
93 {
94 #ifdef CONFIG_ARM_LPAE
95 	return __phys_to_pfn(virt_to_phys(pgd));
96 #else
97 	return virt_to_phys(pgd);
98 #endif
99 }
100 
101 #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
secondary_biglittle_prepare(unsigned int cpu)102 static int secondary_biglittle_prepare(unsigned int cpu)
103 {
104 	if (!cpu_vtable[cpu])
105 		cpu_vtable[cpu] = kzalloc(sizeof(*cpu_vtable[cpu]), GFP_KERNEL);
106 
107 	return cpu_vtable[cpu] ? 0 : -ENOMEM;
108 }
109 
secondary_biglittle_init(void)110 static void secondary_biglittle_init(void)
111 {
112 	init_proc_vtable(lookup_processor(read_cpuid_id())->proc);
113 }
114 #else
secondary_biglittle_prepare(unsigned int cpu)115 static int secondary_biglittle_prepare(unsigned int cpu)
116 {
117 	return 0;
118 }
119 
secondary_biglittle_init(void)120 static void secondary_biglittle_init(void)
121 {
122 }
123 #endif
124 
__cpu_up(unsigned int cpu,struct task_struct * idle)125 int __cpu_up(unsigned int cpu, struct task_struct *idle)
126 {
127 	int ret;
128 
129 	if (!smp_ops.smp_boot_secondary)
130 		return -ENOSYS;
131 
132 	ret = secondary_biglittle_prepare(cpu);
133 	if (ret)
134 		return ret;
135 
136 	/*
137 	 * We need to tell the secondary core where to find
138 	 * its stack and the page tables.
139 	 */
140 	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
141 #ifdef CONFIG_ARM_MPU
142 	secondary_data.mpu_rgn_szr = mpu_rgn_info.rgns[MPU_RAM_REGION].drsr;
143 #endif
144 
145 #ifdef CONFIG_MMU
146 	secondary_data.pgdir = virt_to_phys(idmap_pgd);
147 	secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir);
148 #endif
149 	sync_cache_w(&secondary_data);
150 
151 	/*
152 	 * Now bring the CPU into our world.
153 	 */
154 	ret = smp_ops.smp_boot_secondary(cpu, idle);
155 	if (ret == 0) {
156 		/*
157 		 * CPU was successfully started, wait for it
158 		 * to come online or time out.
159 		 */
160 		wait_for_completion_timeout(&cpu_running,
161 						 msecs_to_jiffies(1000));
162 
163 		if (!cpu_online(cpu)) {
164 			pr_crit("CPU%u: failed to come online\n", cpu);
165 			ret = -EIO;
166 		}
167 	} else {
168 		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
169 	}
170 
171 
172 	memset(&secondary_data, 0, sizeof(secondary_data));
173 	return ret;
174 }
175 
176 /* platform specific SMP operations */
smp_init_cpus(void)177 void __init smp_init_cpus(void)
178 {
179 	if (smp_ops.smp_init_cpus)
180 		smp_ops.smp_init_cpus();
181 }
182 
platform_can_secondary_boot(void)183 int platform_can_secondary_boot(void)
184 {
185 	return !!smp_ops.smp_boot_secondary;
186 }
187 
platform_can_cpu_hotplug(void)188 int platform_can_cpu_hotplug(void)
189 {
190 #ifdef CONFIG_HOTPLUG_CPU
191 	if (smp_ops.cpu_kill)
192 		return 1;
193 #endif
194 
195 	return 0;
196 }
197 
198 #ifdef CONFIG_HOTPLUG_CPU
platform_cpu_kill(unsigned int cpu)199 static int platform_cpu_kill(unsigned int cpu)
200 {
201 	if (smp_ops.cpu_kill)
202 		return smp_ops.cpu_kill(cpu);
203 	return 1;
204 }
205 
platform_cpu_disable(unsigned int cpu)206 static int platform_cpu_disable(unsigned int cpu)
207 {
208 	if (smp_ops.cpu_disable)
209 		return smp_ops.cpu_disable(cpu);
210 
211 	return 0;
212 }
213 
platform_can_hotplug_cpu(unsigned int cpu)214 int platform_can_hotplug_cpu(unsigned int cpu)
215 {
216 	/* cpu_die must be specified to support hotplug */
217 	if (!smp_ops.cpu_die)
218 		return 0;
219 
220 	if (smp_ops.cpu_can_disable)
221 		return smp_ops.cpu_can_disable(cpu);
222 
223 	/*
224 	 * By default, allow disabling all CPUs except the first one,
225 	 * since this is special on a lot of platforms, e.g. because
226 	 * of clock tick interrupts.
227 	 */
228 	return cpu != 0;
229 }
230 
231 /*
232  * __cpu_disable runs on the processor to be shutdown.
233  */
__cpu_disable(void)234 int __cpu_disable(void)
235 {
236 	unsigned int cpu = smp_processor_id();
237 	int ret;
238 
239 	ret = platform_cpu_disable(cpu);
240 	if (ret)
241 		return ret;
242 
243 	/*
244 	 * Take this CPU offline.  Once we clear this, we can't return,
245 	 * and we must not schedule until we're ready to give up the cpu.
246 	 */
247 	set_cpu_online(cpu, false);
248 
249 	/*
250 	 * OK - migrate IRQs away from this CPU
251 	 */
252 	irq_migrate_all_off_this_cpu();
253 
254 	/*
255 	 * Flush user cache and TLB mappings, and then remove this CPU
256 	 * from the vm mask set of all processes.
257 	 *
258 	 * Caches are flushed to the Level of Unification Inner Shareable
259 	 * to write-back dirty lines to unified caches shared by all CPUs.
260 	 */
261 	flush_cache_louis();
262 	local_flush_tlb_all();
263 
264 	clear_tasks_mm_cpumask(cpu);
265 
266 	return 0;
267 }
268 
269 static DECLARE_COMPLETION(cpu_died);
270 
271 /*
272  * called on the thread which is asking for a CPU to be shutdown -
273  * waits until shutdown has completed, or it is timed out.
274  */
__cpu_die(unsigned int cpu)275 void __cpu_die(unsigned int cpu)
276 {
277 	if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) {
278 		pr_err("CPU%u: cpu didn't die\n", cpu);
279 		return;
280 	}
281 	pr_notice("CPU%u: shutdown\n", cpu);
282 
283 	/*
284 	 * platform_cpu_kill() is generally expected to do the powering off
285 	 * and/or cutting of clocks to the dying CPU.  Optionally, this may
286 	 * be done by the CPU which is dying in preference to supporting
287 	 * this call, but that means there is _no_ synchronisation between
288 	 * the requesting CPU and the dying CPU actually losing power.
289 	 */
290 	if (!platform_cpu_kill(cpu))
291 		pr_err("CPU%u: unable to kill\n", cpu);
292 }
293 
294 /*
295  * Called from the idle thread for the CPU which has been shutdown.
296  *
297  * Note that we disable IRQs here, but do not re-enable them
298  * before returning to the caller. This is also the behaviour
299  * of the other hotplug-cpu capable cores, so presumably coming
300  * out of idle fixes this.
301  */
arch_cpu_idle_dead(void)302 void arch_cpu_idle_dead(void)
303 {
304 	unsigned int cpu = smp_processor_id();
305 
306 	idle_task_exit();
307 
308 	local_irq_disable();
309 
310 	/*
311 	 * Flush the data out of the L1 cache for this CPU.  This must be
312 	 * before the completion to ensure that data is safely written out
313 	 * before platform_cpu_kill() gets called - which may disable
314 	 * *this* CPU and power down its cache.
315 	 */
316 	flush_cache_louis();
317 
318 	/*
319 	 * Tell __cpu_die() that this CPU is now safe to dispose of.  Once
320 	 * this returns, power and/or clocks can be removed at any point
321 	 * from this CPU and its cache by platform_cpu_kill().
322 	 */
323 	complete(&cpu_died);
324 
325 	/*
326 	 * Ensure that the cache lines associated with that completion are
327 	 * written out.  This covers the case where _this_ CPU is doing the
328 	 * powering down, to ensure that the completion is visible to the
329 	 * CPU waiting for this one.
330 	 */
331 	flush_cache_louis();
332 
333 	/*
334 	 * The actual CPU shutdown procedure is at least platform (if not
335 	 * CPU) specific.  This may remove power, or it may simply spin.
336 	 *
337 	 * Platforms are generally expected *NOT* to return from this call,
338 	 * although there are some which do because they have no way to
339 	 * power down the CPU.  These platforms are the _only_ reason we
340 	 * have a return path which uses the fragment of assembly below.
341 	 *
342 	 * The return path should not be used for platforms which can
343 	 * power off the CPU.
344 	 */
345 	if (smp_ops.cpu_die)
346 		smp_ops.cpu_die(cpu);
347 
348 	pr_warn("CPU%u: smp_ops.cpu_die() returned, trying to resuscitate\n",
349 		cpu);
350 
351 	/*
352 	 * Do not return to the idle loop - jump back to the secondary
353 	 * cpu initialisation.  There's some initialisation which needs
354 	 * to be repeated to undo the effects of taking the CPU offline.
355 	 */
356 	__asm__("mov	sp, %0\n"
357 	"	mov	fp, #0\n"
358 	"	b	secondary_start_kernel"
359 		:
360 		: "r" (task_stack_page(current) + THREAD_SIZE - 8));
361 }
362 #endif /* CONFIG_HOTPLUG_CPU */
363 
364 /*
365  * Called by both boot and secondaries to move global data into
366  * per-processor storage.
367  */
smp_store_cpu_info(unsigned int cpuid)368 static void smp_store_cpu_info(unsigned int cpuid)
369 {
370 	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
371 
372 	cpu_info->loops_per_jiffy = loops_per_jiffy;
373 	cpu_info->cpuid = read_cpuid_id();
374 
375 	store_cpu_topology(cpuid);
376 }
377 
378 /*
379  * This is the secondary CPU boot entry.  We're using this CPUs
380  * idle thread stack, but a set of temporary page tables.
381  */
secondary_start_kernel(void)382 asmlinkage void secondary_start_kernel(void)
383 {
384 	struct mm_struct *mm = &init_mm;
385 	unsigned int cpu;
386 
387 	secondary_biglittle_init();
388 
389 	/*
390 	 * The identity mapping is uncached (strongly ordered), so
391 	 * switch away from it before attempting any exclusive accesses.
392 	 */
393 	cpu_switch_mm(mm->pgd, mm);
394 	local_flush_bp_all();
395 	enter_lazy_tlb(mm, current);
396 	local_flush_tlb_all();
397 
398 	/*
399 	 * All kernel threads share the same mm context; grab a
400 	 * reference and switch to it.
401 	 */
402 	cpu = smp_processor_id();
403 	atomic_inc(&mm->mm_count);
404 	current->active_mm = mm;
405 	cpumask_set_cpu(cpu, mm_cpumask(mm));
406 
407 	cpu_init();
408 
409 	pr_debug("CPU%u: Booted secondary processor\n", cpu);
410 
411 	preempt_disable();
412 	trace_hardirqs_off();
413 
414 	/*
415 	 * Give the platform a chance to do its own initialisation.
416 	 */
417 	if (smp_ops.smp_secondary_init)
418 		smp_ops.smp_secondary_init(cpu);
419 
420 	notify_cpu_starting(cpu);
421 
422 	calibrate_delay();
423 
424 	smp_store_cpu_info(cpu);
425 
426 	/*
427 	 * OK, now it's safe to let the boot CPU continue.  Wait for
428 	 * the CPU migration code to notice that the CPU is online
429 	 * before we continue - which happens after __cpu_up returns.
430 	 */
431 	set_cpu_online(cpu, true);
432 
433 	check_other_bugs();
434 
435 	complete(&cpu_running);
436 
437 	local_irq_enable();
438 	local_fiq_enable();
439 	local_abt_enable();
440 
441 	/*
442 	 * OK, it's off to the idle thread for us
443 	 */
444 	cpu_startup_entry(CPUHP_ONLINE);
445 }
446 
smp_cpus_done(unsigned int max_cpus)447 void __init smp_cpus_done(unsigned int max_cpus)
448 {
449 	int cpu;
450 	unsigned long bogosum = 0;
451 
452 	for_each_online_cpu(cpu)
453 		bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
454 
455 	printk(KERN_INFO "SMP: Total of %d processors activated "
456 	       "(%lu.%02lu BogoMIPS).\n",
457 	       num_online_cpus(),
458 	       bogosum / (500000/HZ),
459 	       (bogosum / (5000/HZ)) % 100);
460 
461 	hyp_mode_check();
462 }
463 
smp_prepare_boot_cpu(void)464 void __init smp_prepare_boot_cpu(void)
465 {
466 	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
467 }
468 
smp_prepare_cpus(unsigned int max_cpus)469 void __init smp_prepare_cpus(unsigned int max_cpus)
470 {
471 	unsigned int ncores = num_possible_cpus();
472 
473 	init_cpu_topology();
474 
475 	smp_store_cpu_info(smp_processor_id());
476 
477 	/*
478 	 * are we trying to boot more cores than exist?
479 	 */
480 	if (max_cpus > ncores)
481 		max_cpus = ncores;
482 	if (ncores > 1 && max_cpus) {
483 		/*
484 		 * Initialise the present map, which describes the set of CPUs
485 		 * actually populated at the present time. A platform should
486 		 * re-initialize the map in the platforms smp_prepare_cpus()
487 		 * if present != possible (e.g. physical hotplug).
488 		 */
489 		init_cpu_present(cpu_possible_mask);
490 
491 		/*
492 		 * Initialise the SCU if there are more than one CPU
493 		 * and let them know where to start.
494 		 */
495 		if (smp_ops.smp_prepare_cpus)
496 			smp_ops.smp_prepare_cpus(max_cpus);
497 	}
498 }
499 
500 static void (*__smp_cross_call)(const struct cpumask *, unsigned int);
501 
set_smp_cross_call(void (* fn)(const struct cpumask *,unsigned int))502 void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
503 {
504 	if (!__smp_cross_call)
505 		__smp_cross_call = fn;
506 }
507 
508 static const char *ipi_types[NR_IPI] __tracepoint_string = {
509 #define S(x,s)	[x] = s
510 	S(IPI_WAKEUP, "CPU wakeup interrupts"),
511 	S(IPI_TIMER, "Timer broadcast interrupts"),
512 	S(IPI_RESCHEDULE, "Rescheduling interrupts"),
513 	S(IPI_CALL_FUNC, "Function call interrupts"),
514 	S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
515 	S(IPI_CPU_STOP, "CPU stop interrupts"),
516 	S(IPI_IRQ_WORK, "IRQ work interrupts"),
517 	S(IPI_COMPLETION, "completion interrupts"),
518 };
519 
smp_cross_call(const struct cpumask * target,unsigned int ipinr)520 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
521 {
522 	trace_ipi_raise(target, ipi_types[ipinr]);
523 	__smp_cross_call(target, ipinr);
524 }
525 
show_ipi_list(struct seq_file * p,int prec)526 void show_ipi_list(struct seq_file *p, int prec)
527 {
528 	unsigned int cpu, i;
529 
530 	for (i = 0; i < NR_IPI; i++) {
531 		seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
532 
533 		for_each_online_cpu(cpu)
534 			seq_printf(p, "%10u ",
535 				   __get_irq_stat(cpu, ipi_irqs[i]));
536 
537 		seq_printf(p, " %s\n", ipi_types[i]);
538 	}
539 }
540 
smp_irq_stat_cpu(unsigned int cpu)541 u64 smp_irq_stat_cpu(unsigned int cpu)
542 {
543 	u64 sum = 0;
544 	int i;
545 
546 	for (i = 0; i < NR_IPI; i++)
547 		sum += __get_irq_stat(cpu, ipi_irqs[i]);
548 
549 	return sum;
550 }
551 
arch_send_call_function_ipi_mask(const struct cpumask * mask)552 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
553 {
554 	smp_cross_call(mask, IPI_CALL_FUNC);
555 }
556 
arch_send_wakeup_ipi_mask(const struct cpumask * mask)557 void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
558 {
559 	smp_cross_call(mask, IPI_WAKEUP);
560 }
561 
arch_send_call_function_single_ipi(int cpu)562 void arch_send_call_function_single_ipi(int cpu)
563 {
564 	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
565 }
566 
567 #ifdef CONFIG_IRQ_WORK
arch_irq_work_raise(void)568 void arch_irq_work_raise(void)
569 {
570 	if (arch_irq_work_has_interrupt())
571 		smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
572 }
573 #endif
574 
575 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
tick_broadcast(const struct cpumask * mask)576 void tick_broadcast(const struct cpumask *mask)
577 {
578 	smp_cross_call(mask, IPI_TIMER);
579 }
580 #endif
581 
582 static DEFINE_RAW_SPINLOCK(stop_lock);
583 
584 /*
585  * ipi_cpu_stop - handle IPI from smp_send_stop()
586  */
ipi_cpu_stop(unsigned int cpu)587 static void ipi_cpu_stop(unsigned int cpu)
588 {
589 	if (system_state == SYSTEM_BOOTING ||
590 	    system_state == SYSTEM_RUNNING) {
591 		raw_spin_lock(&stop_lock);
592 		pr_crit("CPU%u: stopping\n", cpu);
593 		dump_stack();
594 		raw_spin_unlock(&stop_lock);
595 	}
596 
597 	set_cpu_online(cpu, false);
598 
599 	local_fiq_disable();
600 	local_irq_disable();
601 
602 	while (1) {
603 		cpu_relax();
604 		wfe();
605 	}
606 }
607 
608 static DEFINE_PER_CPU(struct completion *, cpu_completion);
609 
register_ipi_completion(struct completion * completion,int cpu)610 int register_ipi_completion(struct completion *completion, int cpu)
611 {
612 	per_cpu(cpu_completion, cpu) = completion;
613 	return IPI_COMPLETION;
614 }
615 
ipi_complete(unsigned int cpu)616 static void ipi_complete(unsigned int cpu)
617 {
618 	complete(per_cpu(cpu_completion, cpu));
619 }
620 
621 /*
622  * Main handler for inter-processor interrupts
623  */
do_IPI(int ipinr,struct pt_regs * regs)624 asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs)
625 {
626 	handle_IPI(ipinr, regs);
627 }
628 
handle_IPI(int ipinr,struct pt_regs * regs)629 void handle_IPI(int ipinr, struct pt_regs *regs)
630 {
631 	unsigned int cpu = smp_processor_id();
632 	struct pt_regs *old_regs = set_irq_regs(regs);
633 
634 	if ((unsigned)ipinr < NR_IPI) {
635 		trace_ipi_entry_rcuidle(ipi_types[ipinr]);
636 		__inc_irq_stat(cpu, ipi_irqs[ipinr]);
637 	}
638 
639 	switch (ipinr) {
640 	case IPI_WAKEUP:
641 		break;
642 
643 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
644 	case IPI_TIMER:
645 		irq_enter();
646 		tick_receive_broadcast();
647 		irq_exit();
648 		break;
649 #endif
650 
651 	case IPI_RESCHEDULE:
652 		scheduler_ipi();
653 		break;
654 
655 	case IPI_CALL_FUNC:
656 		irq_enter();
657 		generic_smp_call_function_interrupt();
658 		irq_exit();
659 		break;
660 
661 	case IPI_CALL_FUNC_SINGLE:
662 		irq_enter();
663 		generic_smp_call_function_single_interrupt();
664 		irq_exit();
665 		break;
666 
667 	case IPI_CPU_STOP:
668 		irq_enter();
669 		ipi_cpu_stop(cpu);
670 		irq_exit();
671 		break;
672 
673 #ifdef CONFIG_IRQ_WORK
674 	case IPI_IRQ_WORK:
675 		irq_enter();
676 		irq_work_run();
677 		irq_exit();
678 		break;
679 #endif
680 
681 	case IPI_COMPLETION:
682 		irq_enter();
683 		ipi_complete(cpu);
684 		irq_exit();
685 		break;
686 
687 	case IPI_CPU_BACKTRACE:
688 		irq_enter();
689 		nmi_cpu_backtrace(regs);
690 		irq_exit();
691 		break;
692 
693 	default:
694 		pr_crit("CPU%u: Unknown IPI message 0x%x\n",
695 		        cpu, ipinr);
696 		break;
697 	}
698 
699 	if ((unsigned)ipinr < NR_IPI)
700 		trace_ipi_exit_rcuidle(ipi_types[ipinr]);
701 	set_irq_regs(old_regs);
702 }
703 
smp_send_reschedule(int cpu)704 void smp_send_reschedule(int cpu)
705 {
706 	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
707 }
708 
smp_send_stop(void)709 void smp_send_stop(void)
710 {
711 	unsigned long timeout;
712 	struct cpumask mask;
713 
714 	cpumask_copy(&mask, cpu_online_mask);
715 	cpumask_clear_cpu(smp_processor_id(), &mask);
716 	if (!cpumask_empty(&mask))
717 		smp_cross_call(&mask, IPI_CPU_STOP);
718 
719 	/* Wait up to one second for other CPUs to stop */
720 	timeout = USEC_PER_SEC;
721 	while (num_online_cpus() > 1 && timeout--)
722 		udelay(1);
723 
724 	if (num_online_cpus() > 1)
725 		pr_warn("SMP: failed to stop secondary CPUs\n");
726 }
727 
728 /* In case panic() and panic() called at the same time on CPU1 and CPU2,
729  * and CPU 1 calls panic_smp_self_stop() before crash_smp_send_stop()
730  * CPU1 can't receive the ipi irqs from CPU2, CPU1 will be always online,
731  * kdump fails. So split out the panic_smp_self_stop() and add
732  * set_cpu_online(smp_processor_id(), false).
733  */
panic_smp_self_stop(void)734 void panic_smp_self_stop(void)
735 {
736 	pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n",
737 	         smp_processor_id());
738 	set_cpu_online(smp_processor_id(), false);
739 	while (1)
740 		cpu_relax();
741 }
742 
743 /*
744  * not supported here
745  */
setup_profiling_timer(unsigned int multiplier)746 int setup_profiling_timer(unsigned int multiplier)
747 {
748 	return -EINVAL;
749 }
750 
751 #ifdef CONFIG_CPU_FREQ
752 
753 static DEFINE_PER_CPU(unsigned long, l_p_j_ref);
754 static DEFINE_PER_CPU(unsigned long, l_p_j_ref_freq);
755 static unsigned long global_l_p_j_ref;
756 static unsigned long global_l_p_j_ref_freq;
757 
cpufreq_callback(struct notifier_block * nb,unsigned long val,void * data)758 static int cpufreq_callback(struct notifier_block *nb,
759 					unsigned long val, void *data)
760 {
761 	struct cpufreq_freqs *freq = data;
762 	int cpu = freq->cpu;
763 
764 	if (freq->flags & CPUFREQ_CONST_LOOPS)
765 		return NOTIFY_OK;
766 
767 	if (!per_cpu(l_p_j_ref, cpu)) {
768 		per_cpu(l_p_j_ref, cpu) =
769 			per_cpu(cpu_data, cpu).loops_per_jiffy;
770 		per_cpu(l_p_j_ref_freq, cpu) = freq->old;
771 		if (!global_l_p_j_ref) {
772 			global_l_p_j_ref = loops_per_jiffy;
773 			global_l_p_j_ref_freq = freq->old;
774 		}
775 	}
776 
777 	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
778 	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {
779 		loops_per_jiffy = cpufreq_scale(global_l_p_j_ref,
780 						global_l_p_j_ref_freq,
781 						freq->new);
782 		per_cpu(cpu_data, cpu).loops_per_jiffy =
783 			cpufreq_scale(per_cpu(l_p_j_ref, cpu),
784 					per_cpu(l_p_j_ref_freq, cpu),
785 					freq->new);
786 	}
787 	return NOTIFY_OK;
788 }
789 
790 static struct notifier_block cpufreq_notifier = {
791 	.notifier_call  = cpufreq_callback,
792 };
793 
register_cpufreq_notifier(void)794 static int __init register_cpufreq_notifier(void)
795 {
796 	return cpufreq_register_notifier(&cpufreq_notifier,
797 						CPUFREQ_TRANSITION_NOTIFIER);
798 }
799 core_initcall(register_cpufreq_notifier);
800 
801 #endif
802 
raise_nmi(cpumask_t * mask)803 static void raise_nmi(cpumask_t *mask)
804 {
805 	/*
806 	 * Generate the backtrace directly if we are running in a calling
807 	 * context that is not preemptible by the backtrace IPI. Note
808 	 * that nmi_cpu_backtrace() automatically removes the current cpu
809 	 * from mask.
810 	 */
811 	if (cpumask_test_cpu(smp_processor_id(), mask) && irqs_disabled())
812 		nmi_cpu_backtrace(NULL);
813 
814 	smp_cross_call(mask, IPI_CPU_BACKTRACE);
815 }
816 
arch_trigger_all_cpu_backtrace(bool include_self)817 void arch_trigger_all_cpu_backtrace(bool include_self)
818 {
819 	nmi_trigger_all_cpu_backtrace(include_self, raise_nmi);
820 }
821