• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
4  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
6  *
7  *  No idle tick implementation for low and high resolution timers
8  *
9  *  Started by: Thomas Gleixner and Ingo Molnar
10  */
11 #include <linux/cpu.h>
12 #include <linux/err.h>
13 #include <linux/hrtimer.h>
14 #include <linux/interrupt.h>
15 #include <linux/kernel_stat.h>
16 #include <linux/percpu.h>
17 #include <linux/nmi.h>
18 #include <linux/profile.h>
19 #include <linux/sched/signal.h>
20 #include <linux/sched/clock.h>
21 #include <linux/sched/stat.h>
22 #include <linux/sched/nohz.h>
23 #include <linux/module.h>
24 #include <linux/irq_work.h>
25 #include <linux/posix-timers.h>
26 #include <linux/context_tracking.h>
27 #include <linux/mm.h>
28 
29 #include <asm/irq_regs.h>
30 
31 #include "tick-internal.h"
32 
33 #include <trace/events/timer.h>
34 
35 /*
36  * Per-CPU nohz control structure
37  */
38 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
39 
tick_get_tick_sched(int cpu)40 struct tick_sched *tick_get_tick_sched(int cpu)
41 {
42 	return &per_cpu(tick_cpu_sched, cpu);
43 }
44 
45 #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
46 /*
47  * The time, when the last jiffy update happened. Protected by jiffies_lock.
48  */
49 static ktime_t last_jiffies_update;
50 
51 /*
52  * Must be called with interrupts disabled !
53  */
tick_do_update_jiffies64(ktime_t now)54 static void tick_do_update_jiffies64(ktime_t now)
55 {
56 	unsigned long ticks = 0;
57 	ktime_t delta;
58 
59 	/*
60 	 * Do a quick check without holding jiffies_lock:
61 	 * The READ_ONCE() pairs with two updates done later in this function.
62 	 */
63 	delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
64 	if (delta < tick_period)
65 		return;
66 
67 	/* Reevaluate with jiffies_lock held */
68 	raw_spin_lock(&jiffies_lock);
69 	write_seqcount_begin(&jiffies_seq);
70 
71 	delta = ktime_sub(now, last_jiffies_update);
72 	if (delta >= tick_period) {
73 
74 		delta = ktime_sub(delta, tick_period);
75 		/* Pairs with the lockless read in this function. */
76 		WRITE_ONCE(last_jiffies_update,
77 			   ktime_add(last_jiffies_update, tick_period));
78 
79 		/* Slow path for long timeouts */
80 		if (unlikely(delta >= tick_period)) {
81 			s64 incr = ktime_to_ns(tick_period);
82 
83 			ticks = ktime_divns(delta, incr);
84 
85 			/* Pairs with the lockless read in this function. */
86 			WRITE_ONCE(last_jiffies_update,
87 				   ktime_add_ns(last_jiffies_update,
88 						incr * ticks));
89 		}
90 		do_timer(++ticks);
91 
92 		/* Keep the tick_next_period variable up to date */
93 		tick_next_period = ktime_add(last_jiffies_update, tick_period);
94 	} else {
95 		write_seqcount_end(&jiffies_seq);
96 		raw_spin_unlock(&jiffies_lock);
97 		return;
98 	}
99 	write_seqcount_end(&jiffies_seq);
100 	raw_spin_unlock(&jiffies_lock);
101 	update_wall_time();
102 }
103 
104 /*
105  * Initialize and return retrieve the jiffies update.
106  */
tick_init_jiffy_update(void)107 static ktime_t tick_init_jiffy_update(void)
108 {
109 	ktime_t period;
110 
111 	raw_spin_lock(&jiffies_lock);
112 	write_seqcount_begin(&jiffies_seq);
113 	/* Did we start the jiffies update yet ? */
114 	if (last_jiffies_update == 0)
115 		last_jiffies_update = tick_next_period;
116 	period = last_jiffies_update;
117 	write_seqcount_end(&jiffies_seq);
118 	raw_spin_unlock(&jiffies_lock);
119 	return period;
120 }
121 
tick_sched_do_timer(struct tick_sched * ts,ktime_t now)122 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
123 {
124 	int cpu = smp_processor_id();
125 
126 #ifdef CONFIG_NO_HZ_COMMON
127 	/*
128 	 * Check if the do_timer duty was dropped. We don't care about
129 	 * concurrency: This happens only when the CPU in charge went
130 	 * into a long sleep. If two CPUs happen to assign themselves to
131 	 * this duty, then the jiffies update is still serialized by
132 	 * jiffies_lock.
133 	 *
134 	 * If nohz_full is enabled, this should not happen because the
135 	 * tick_do_timer_cpu never relinquishes.
136 	 */
137 	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
138 #ifdef CONFIG_NO_HZ_FULL
139 		WARN_ON_ONCE(tick_nohz_full_running);
140 #endif
141 		tick_do_timer_cpu = cpu;
142 	}
143 #endif
144 
145 	/* Check, if the jiffies need an update */
146 	if (tick_do_timer_cpu == cpu)
147 		tick_do_update_jiffies64(now);
148 
149 	if (ts->inidle)
150 		ts->got_idle_tick = 1;
151 }
152 
tick_sched_handle(struct tick_sched * ts,struct pt_regs * regs)153 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
154 {
155 #ifdef CONFIG_NO_HZ_COMMON
156 	/*
157 	 * When we are idle and the tick is stopped, we have to touch
158 	 * the watchdog as we might not schedule for a really long
159 	 * time. This happens on complete idle SMP systems while
160 	 * waiting on the login prompt. We also increment the "start of
161 	 * idle" jiffy stamp so the idle accounting adjustment we do
162 	 * when we go busy again does not account too much ticks.
163 	 */
164 	if (ts->tick_stopped) {
165 		touch_softlockup_watchdog_sched();
166 		if (is_idle_task(current))
167 			ts->idle_jiffies++;
168 		/*
169 		 * In case the current tick fired too early past its expected
170 		 * expiration, make sure we don't bypass the next clock reprogramming
171 		 * to the same deadline.
172 		 */
173 		ts->next_tick = 0;
174 	}
175 #endif
176 	update_process_times(user_mode(regs));
177 	profile_tick(CPU_PROFILING);
178 }
179 #endif
180 
181 #ifdef CONFIG_NO_HZ_FULL
182 cpumask_var_t tick_nohz_full_mask;
183 bool tick_nohz_full_running;
184 EXPORT_SYMBOL_GPL(tick_nohz_full_running);
185 static atomic_t tick_dep_mask;
186 
check_tick_dependency(atomic_t * dep)187 static bool check_tick_dependency(atomic_t *dep)
188 {
189 	int val = atomic_read(dep);
190 
191 	if (val & TICK_DEP_MASK_POSIX_TIMER) {
192 		trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
193 		return true;
194 	}
195 
196 	if (val & TICK_DEP_MASK_PERF_EVENTS) {
197 		trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
198 		return true;
199 	}
200 
201 	if (val & TICK_DEP_MASK_SCHED) {
202 		trace_tick_stop(0, TICK_DEP_MASK_SCHED);
203 		return true;
204 	}
205 
206 	if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
207 		trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
208 		return true;
209 	}
210 
211 	if (val & TICK_DEP_MASK_RCU) {
212 		trace_tick_stop(0, TICK_DEP_MASK_RCU);
213 		return true;
214 	}
215 
216 	return false;
217 }
218 
can_stop_full_tick(int cpu,struct tick_sched * ts)219 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
220 {
221 	lockdep_assert_irqs_disabled();
222 
223 	if (unlikely(!cpu_online(cpu)))
224 		return false;
225 
226 	if (check_tick_dependency(&tick_dep_mask))
227 		return false;
228 
229 	if (check_tick_dependency(&ts->tick_dep_mask))
230 		return false;
231 
232 	if (check_tick_dependency(&current->tick_dep_mask))
233 		return false;
234 
235 	if (check_tick_dependency(&current->signal->tick_dep_mask))
236 		return false;
237 
238 	return true;
239 }
240 
nohz_full_kick_func(struct irq_work * work)241 static void nohz_full_kick_func(struct irq_work *work)
242 {
243 	/* Empty, the tick restart happens on tick_nohz_irq_exit() */
244 }
245 
246 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
247 	.func = nohz_full_kick_func,
248 	.flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
249 };
250 
251 /*
252  * Kick this CPU if it's full dynticks in order to force it to
253  * re-evaluate its dependency on the tick and restart it if necessary.
254  * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
255  * is NMI safe.
256  */
tick_nohz_full_kick(void)257 static void tick_nohz_full_kick(void)
258 {
259 	if (!tick_nohz_full_cpu(smp_processor_id()))
260 		return;
261 
262 	irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
263 }
264 
265 /*
266  * Kick the CPU if it's full dynticks in order to force it to
267  * re-evaluate its dependency on the tick and restart it if necessary.
268  */
tick_nohz_full_kick_cpu(int cpu)269 void tick_nohz_full_kick_cpu(int cpu)
270 {
271 	if (!tick_nohz_full_cpu(cpu))
272 		return;
273 
274 	irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
275 }
276 
277 /*
278  * Kick all full dynticks CPUs in order to force these to re-evaluate
279  * their dependency on the tick and restart it if necessary.
280  */
tick_nohz_full_kick_all(void)281 static void tick_nohz_full_kick_all(void)
282 {
283 	int cpu;
284 
285 	if (!tick_nohz_full_running)
286 		return;
287 
288 	preempt_disable();
289 	for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
290 		tick_nohz_full_kick_cpu(cpu);
291 	preempt_enable();
292 }
293 
tick_nohz_dep_set_all(atomic_t * dep,enum tick_dep_bits bit)294 static void tick_nohz_dep_set_all(atomic_t *dep,
295 				  enum tick_dep_bits bit)
296 {
297 	int prev;
298 
299 	prev = atomic_fetch_or(BIT(bit), dep);
300 	if (!prev)
301 		tick_nohz_full_kick_all();
302 }
303 
304 /*
305  * Set a global tick dependency. Used by perf events that rely on freq and
306  * by unstable clock.
307  */
tick_nohz_dep_set(enum tick_dep_bits bit)308 void tick_nohz_dep_set(enum tick_dep_bits bit)
309 {
310 	tick_nohz_dep_set_all(&tick_dep_mask, bit);
311 }
312 
tick_nohz_dep_clear(enum tick_dep_bits bit)313 void tick_nohz_dep_clear(enum tick_dep_bits bit)
314 {
315 	atomic_andnot(BIT(bit), &tick_dep_mask);
316 }
317 
318 /*
319  * Set per-CPU tick dependency. Used by scheduler and perf events in order to
320  * manage events throttling.
321  */
tick_nohz_dep_set_cpu(int cpu,enum tick_dep_bits bit)322 void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
323 {
324 	int prev;
325 	struct tick_sched *ts;
326 
327 	ts = per_cpu_ptr(&tick_cpu_sched, cpu);
328 
329 	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
330 	if (!prev) {
331 		preempt_disable();
332 		/* Perf needs local kick that is NMI safe */
333 		if (cpu == smp_processor_id()) {
334 			tick_nohz_full_kick();
335 		} else {
336 			/* Remote irq work not NMI-safe */
337 			if (!WARN_ON_ONCE(in_nmi()))
338 				tick_nohz_full_kick_cpu(cpu);
339 		}
340 		preempt_enable();
341 	}
342 }
343 EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
344 
tick_nohz_dep_clear_cpu(int cpu,enum tick_dep_bits bit)345 void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
346 {
347 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
348 
349 	atomic_andnot(BIT(bit), &ts->tick_dep_mask);
350 }
351 EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
352 
353 /*
354  * Set a per-task tick dependency. RCU need this. Also posix CPU timers
355  * in order to elapse per task timers.
356  */
tick_nohz_dep_set_task(struct task_struct * tsk,enum tick_dep_bits bit)357 void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
358 {
359 	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
360 		if (tsk == current) {
361 			preempt_disable();
362 			tick_nohz_full_kick();
363 			preempt_enable();
364 		} else {
365 			/*
366 			 * Some future tick_nohz_full_kick_task()
367 			 * should optimize this.
368 			 */
369 			tick_nohz_full_kick_all();
370 		}
371 	}
372 }
373 EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
374 
tick_nohz_dep_clear_task(struct task_struct * tsk,enum tick_dep_bits bit)375 void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
376 {
377 	atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
378 }
379 EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
380 
381 /*
382  * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
383  * per process timers.
384  */
tick_nohz_dep_set_signal(struct signal_struct * sig,enum tick_dep_bits bit)385 void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
386 {
387 	tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
388 }
389 
tick_nohz_dep_clear_signal(struct signal_struct * sig,enum tick_dep_bits bit)390 void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
391 {
392 	atomic_andnot(BIT(bit), &sig->tick_dep_mask);
393 }
394 
395 /*
396  * Re-evaluate the need for the tick as we switch the current task.
397  * It might need the tick due to per task/process properties:
398  * perf events, posix CPU timers, ...
399  */
__tick_nohz_task_switch(void)400 void __tick_nohz_task_switch(void)
401 {
402 	unsigned long flags;
403 	struct tick_sched *ts;
404 
405 	local_irq_save(flags);
406 
407 	if (!tick_nohz_full_cpu(smp_processor_id()))
408 		goto out;
409 
410 	ts = this_cpu_ptr(&tick_cpu_sched);
411 
412 	if (ts->tick_stopped) {
413 		if (atomic_read(&current->tick_dep_mask) ||
414 		    atomic_read(&current->signal->tick_dep_mask))
415 			tick_nohz_full_kick();
416 	}
417 out:
418 	local_irq_restore(flags);
419 }
420 
421 /* Get the boot-time nohz CPU list from the kernel parameters. */
tick_nohz_full_setup(cpumask_var_t cpumask)422 void __init tick_nohz_full_setup(cpumask_var_t cpumask)
423 {
424 	alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
425 	cpumask_copy(tick_nohz_full_mask, cpumask);
426 	tick_nohz_full_running = true;
427 }
428 
tick_nohz_cpu_down(unsigned int cpu)429 static int tick_nohz_cpu_down(unsigned int cpu)
430 {
431 	/*
432 	 * The tick_do_timer_cpu CPU handles housekeeping duty (unbound
433 	 * timers, workqueues, timekeeping, ...) on behalf of full dynticks
434 	 * CPUs. It must remain online when nohz full is enabled.
435 	 */
436 	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
437 		return -EBUSY;
438 	return 0;
439 }
440 
tick_nohz_init(void)441 void __init tick_nohz_init(void)
442 {
443 	int cpu, ret;
444 
445 	if (!tick_nohz_full_running)
446 		return;
447 
448 	/*
449 	 * Full dynticks uses irq work to drive the tick rescheduling on safe
450 	 * locking contexts. But then we need irq work to raise its own
451 	 * interrupts to avoid circular dependency on the tick
452 	 */
453 	if (!arch_irq_work_has_interrupt()) {
454 		pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
455 		cpumask_clear(tick_nohz_full_mask);
456 		tick_nohz_full_running = false;
457 		return;
458 	}
459 
460 	if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
461 			!IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
462 		cpu = smp_processor_id();
463 
464 		if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
465 			pr_warn("NO_HZ: Clearing %d from nohz_full range "
466 				"for timekeeping\n", cpu);
467 			cpumask_clear_cpu(cpu, tick_nohz_full_mask);
468 		}
469 	}
470 
471 	for_each_cpu(cpu, tick_nohz_full_mask)
472 		context_tracking_cpu_set(cpu);
473 
474 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
475 					"kernel/nohz:predown", NULL,
476 					tick_nohz_cpu_down);
477 	WARN_ON(ret < 0);
478 	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
479 		cpumask_pr_args(tick_nohz_full_mask));
480 }
481 #endif
482 
483 /*
484  * NOHZ - aka dynamic tick functionality
485  */
486 #ifdef CONFIG_NO_HZ_COMMON
487 /*
488  * NO HZ enabled ?
489  */
490 bool tick_nohz_enabled __read_mostly  = true;
491 unsigned long tick_nohz_active  __read_mostly;
492 /*
493  * Enable / Disable tickless mode
494  */
setup_tick_nohz(char * str)495 static int __init setup_tick_nohz(char *str)
496 {
497 	return (kstrtobool(str, &tick_nohz_enabled) == 0);
498 }
499 
500 __setup("nohz=", setup_tick_nohz);
501 
tick_nohz_tick_stopped(void)502 bool tick_nohz_tick_stopped(void)
503 {
504 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
505 
506 	return ts->tick_stopped;
507 }
508 
tick_nohz_tick_stopped_cpu(int cpu)509 bool tick_nohz_tick_stopped_cpu(int cpu)
510 {
511 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
512 
513 	return ts->tick_stopped;
514 }
515 
516 /**
517  * tick_nohz_update_jiffies - update jiffies when idle was interrupted
518  *
519  * Called from interrupt entry when the CPU was idle
520  *
521  * In case the sched_tick was stopped on this CPU, we have to check if jiffies
522  * must be updated. Otherwise an interrupt handler could use a stale jiffy
523  * value. We do this unconditionally on any CPU, as we don't know whether the
524  * CPU, which has the update task assigned is in a long sleep.
525  */
tick_nohz_update_jiffies(ktime_t now)526 static void tick_nohz_update_jiffies(ktime_t now)
527 {
528 	unsigned long flags;
529 
530 	__this_cpu_write(tick_cpu_sched.idle_waketime, now);
531 
532 	local_irq_save(flags);
533 	tick_do_update_jiffies64(now);
534 	local_irq_restore(flags);
535 
536 	touch_softlockup_watchdog_sched();
537 }
538 
539 /*
540  * Updates the per-CPU time idle statistics counters
541  */
542 static void
update_ts_time_stats(int cpu,struct tick_sched * ts,ktime_t now,u64 * last_update_time)543 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
544 {
545 	ktime_t delta;
546 
547 	if (ts->idle_active) {
548 		delta = ktime_sub(now, ts->idle_entrytime);
549 		if (nr_iowait_cpu(cpu) > 0)
550 			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
551 		else
552 			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
553 		ts->idle_entrytime = now;
554 	}
555 
556 	if (last_update_time)
557 		*last_update_time = ktime_to_us(now);
558 
559 }
560 
tick_nohz_stop_idle(struct tick_sched * ts,ktime_t now)561 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
562 {
563 	update_ts_time_stats(smp_processor_id(), ts, now, NULL);
564 	ts->idle_active = 0;
565 
566 	sched_clock_idle_wakeup_event();
567 }
568 
tick_nohz_start_idle(struct tick_sched * ts)569 static void tick_nohz_start_idle(struct tick_sched *ts)
570 {
571 	ts->idle_entrytime = ktime_get();
572 	ts->idle_active = 1;
573 	sched_clock_idle_sleep_event();
574 }
575 
576 /**
577  * get_cpu_idle_time_us - get the total idle time of a CPU
578  * @cpu: CPU number to query
579  * @last_update_time: variable to store update time in. Do not update
580  * counters if NULL.
581  *
582  * Return the cumulative idle time (since boot) for a given
583  * CPU, in microseconds.
584  *
585  * This time is measured via accounting rather than sampling,
586  * and is as accurate as ktime_get() is.
587  *
588  * This function returns -1 if NOHZ is not enabled.
589  */
get_cpu_idle_time_us(int cpu,u64 * last_update_time)590 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
591 {
592 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
593 	ktime_t now, idle;
594 
595 	if (!tick_nohz_active)
596 		return -1;
597 
598 	now = ktime_get();
599 	if (last_update_time) {
600 		update_ts_time_stats(cpu, ts, now, last_update_time);
601 		idle = ts->idle_sleeptime;
602 	} else {
603 		if (ts->idle_active && !nr_iowait_cpu(cpu)) {
604 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
605 
606 			idle = ktime_add(ts->idle_sleeptime, delta);
607 		} else {
608 			idle = ts->idle_sleeptime;
609 		}
610 	}
611 
612 	return ktime_to_us(idle);
613 
614 }
615 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
616 
617 /**
618  * get_cpu_iowait_time_us - get the total iowait time of a CPU
619  * @cpu: CPU number to query
620  * @last_update_time: variable to store update time in. Do not update
621  * counters if NULL.
622  *
623  * Return the cumulative iowait time (since boot) for a given
624  * CPU, in microseconds.
625  *
626  * This time is measured via accounting rather than sampling,
627  * and is as accurate as ktime_get() is.
628  *
629  * This function returns -1 if NOHZ is not enabled.
630  */
get_cpu_iowait_time_us(int cpu,u64 * last_update_time)631 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
632 {
633 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
634 	ktime_t now, iowait;
635 
636 	if (!tick_nohz_active)
637 		return -1;
638 
639 	now = ktime_get();
640 	if (last_update_time) {
641 		update_ts_time_stats(cpu, ts, now, last_update_time);
642 		iowait = ts->iowait_sleeptime;
643 	} else {
644 		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
645 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);
646 
647 			iowait = ktime_add(ts->iowait_sleeptime, delta);
648 		} else {
649 			iowait = ts->iowait_sleeptime;
650 		}
651 	}
652 
653 	return ktime_to_us(iowait);
654 }
655 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
656 
tick_nohz_restart(struct tick_sched * ts,ktime_t now)657 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
658 {
659 	hrtimer_cancel(&ts->sched_timer);
660 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
661 
662 	/* Forward the time to expire in the future */
663 	hrtimer_forward(&ts->sched_timer, now, tick_period);
664 
665 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
666 		hrtimer_start_expires(&ts->sched_timer,
667 				      HRTIMER_MODE_ABS_PINNED_HARD);
668 	} else {
669 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
670 	}
671 
672 	/*
673 	 * Reset to make sure next tick stop doesn't get fooled by past
674 	 * cached clock deadline.
675 	 */
676 	ts->next_tick = 0;
677 }
678 
local_timer_softirq_pending(void)679 static inline bool local_timer_softirq_pending(void)
680 {
681 	return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
682 }
683 
tick_nohz_next_event(struct tick_sched * ts,int cpu)684 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
685 {
686 	u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
687 	unsigned long basejiff;
688 	unsigned int seq;
689 
690 	/* Read jiffies and the time when jiffies were updated last */
691 	do {
692 		seq = read_seqcount_begin(&jiffies_seq);
693 		basemono = last_jiffies_update;
694 		basejiff = jiffies;
695 	} while (read_seqcount_retry(&jiffies_seq, seq));
696 	ts->last_jiffies = basejiff;
697 	ts->timer_expires_base = basemono;
698 
699 	/*
700 	 * Keep the periodic tick, when RCU, architecture or irq_work
701 	 * requests it.
702 	 * Aside of that check whether the local timer softirq is
703 	 * pending. If so its a bad idea to call get_next_timer_interrupt()
704 	 * because there is an already expired timer, so it will request
705 	 * immeditate expiry, which rearms the hardware timer with a
706 	 * minimal delta which brings us back to this place
707 	 * immediately. Lather, rinse and repeat...
708 	 */
709 	if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
710 	    irq_work_needs_cpu() || local_timer_softirq_pending()) {
711 		next_tick = basemono + TICK_NSEC;
712 	} else {
713 		/*
714 		 * Get the next pending timer. If high resolution
715 		 * timers are enabled this only takes the timer wheel
716 		 * timers into account. If high resolution timers are
717 		 * disabled this also looks at the next expiring
718 		 * hrtimer.
719 		 */
720 		next_tmr = get_next_timer_interrupt(basejiff, basemono);
721 		ts->next_timer = next_tmr;
722 		/* Take the next rcu event into account */
723 		next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
724 	}
725 
726 	/*
727 	 * If the tick is due in the next period, keep it ticking or
728 	 * force prod the timer.
729 	 */
730 	delta = next_tick - basemono;
731 	if (delta <= (u64)TICK_NSEC) {
732 		/*
733 		 * Tell the timer code that the base is not idle, i.e. undo
734 		 * the effect of get_next_timer_interrupt():
735 		 */
736 		timer_clear_idle();
737 		/*
738 		 * We've not stopped the tick yet, and there's a timer in the
739 		 * next period, so no point in stopping it either, bail.
740 		 */
741 		if (!ts->tick_stopped) {
742 			ts->timer_expires = 0;
743 			goto out;
744 		}
745 	}
746 
747 	/*
748 	 * If this CPU is the one which had the do_timer() duty last, we limit
749 	 * the sleep time to the timekeeping max_deferment value.
750 	 * Otherwise we can sleep as long as we want.
751 	 */
752 	delta = timekeeping_max_deferment();
753 	if (cpu != tick_do_timer_cpu &&
754 	    (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
755 		delta = KTIME_MAX;
756 
757 	/* Calculate the next expiry time */
758 	if (delta < (KTIME_MAX - basemono))
759 		expires = basemono + delta;
760 	else
761 		expires = KTIME_MAX;
762 
763 	ts->timer_expires = min_t(u64, expires, next_tick);
764 
765 out:
766 	return ts->timer_expires;
767 }
768 
tick_nohz_stop_tick(struct tick_sched * ts,int cpu)769 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
770 {
771 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
772 	u64 basemono = ts->timer_expires_base;
773 	u64 expires = ts->timer_expires;
774 	ktime_t tick = expires;
775 
776 	/* Make sure we won't be trying to stop it twice in a row. */
777 	ts->timer_expires_base = 0;
778 
779 	/*
780 	 * If this CPU is the one which updates jiffies, then give up
781 	 * the assignment and let it be taken by the CPU which runs
782 	 * the tick timer next, which might be this CPU as well. If we
783 	 * don't drop this here the jiffies might be stale and
784 	 * do_timer() never invoked. Keep track of the fact that it
785 	 * was the one which had the do_timer() duty last.
786 	 */
787 	if (cpu == tick_do_timer_cpu) {
788 		tick_do_timer_cpu = TICK_DO_TIMER_NONE;
789 		ts->do_timer_last = 1;
790 	} else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
791 		ts->do_timer_last = 0;
792 	}
793 
794 	/* Skip reprogram of event if its not changed */
795 	if (ts->tick_stopped && (expires == ts->next_tick)) {
796 		/* Sanity check: make sure clockevent is actually programmed */
797 		if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
798 			return;
799 
800 		WARN_ON_ONCE(1);
801 		printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
802 			    basemono, ts->next_tick, dev->next_event,
803 			    hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
804 	}
805 
806 	/*
807 	 * nohz_stop_sched_tick can be called several times before
808 	 * the nohz_restart_sched_tick is called. This happens when
809 	 * interrupts arrive which do not cause a reschedule. In the
810 	 * first call we save the current tick time, so we can restart
811 	 * the scheduler tick in nohz_restart_sched_tick.
812 	 */
813 	if (!ts->tick_stopped) {
814 		calc_load_nohz_start();
815 		quiet_vmstat();
816 
817 		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
818 		ts->tick_stopped = 1;
819 		trace_tick_stop(1, TICK_DEP_MASK_NONE);
820 	}
821 
822 	ts->next_tick = tick;
823 
824 	/*
825 	 * If the expiration time == KTIME_MAX, then we simply stop
826 	 * the tick timer.
827 	 */
828 	if (unlikely(expires == KTIME_MAX)) {
829 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
830 			hrtimer_cancel(&ts->sched_timer);
831 		return;
832 	}
833 
834 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
835 		hrtimer_start(&ts->sched_timer, tick,
836 			      HRTIMER_MODE_ABS_PINNED_HARD);
837 	} else {
838 		hrtimer_set_expires(&ts->sched_timer, tick);
839 		tick_program_event(tick, 1);
840 	}
841 }
842 
tick_nohz_retain_tick(struct tick_sched * ts)843 static void tick_nohz_retain_tick(struct tick_sched *ts)
844 {
845 	ts->timer_expires_base = 0;
846 }
847 
848 #ifdef CONFIG_NO_HZ_FULL
tick_nohz_stop_sched_tick(struct tick_sched * ts,int cpu)849 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
850 {
851 	if (tick_nohz_next_event(ts, cpu))
852 		tick_nohz_stop_tick(ts, cpu);
853 	else
854 		tick_nohz_retain_tick(ts);
855 }
856 #endif /* CONFIG_NO_HZ_FULL */
857 
tick_nohz_restart_sched_tick(struct tick_sched * ts,ktime_t now)858 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
859 {
860 	/* Update jiffies first */
861 	tick_do_update_jiffies64(now);
862 
863 	/*
864 	 * Clear the timer idle flag, so we avoid IPIs on remote queueing and
865 	 * the clock forward checks in the enqueue path:
866 	 */
867 	timer_clear_idle();
868 
869 	calc_load_nohz_stop();
870 	touch_softlockup_watchdog_sched();
871 	/*
872 	 * Cancel the scheduled timer and restore the tick
873 	 */
874 	ts->tick_stopped  = 0;
875 	ts->idle_exittime = now;
876 
877 	tick_nohz_restart(ts, now);
878 }
879 
tick_nohz_full_update_tick(struct tick_sched * ts)880 static void tick_nohz_full_update_tick(struct tick_sched *ts)
881 {
882 #ifdef CONFIG_NO_HZ_FULL
883 	int cpu = smp_processor_id();
884 
885 	if (!tick_nohz_full_cpu(cpu))
886 		return;
887 
888 	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
889 		return;
890 
891 	if (can_stop_full_tick(cpu, ts))
892 		tick_nohz_stop_sched_tick(ts, cpu);
893 	else if (ts->tick_stopped)
894 		tick_nohz_restart_sched_tick(ts, ktime_get());
895 #endif
896 }
897 
can_stop_idle_tick(int cpu,struct tick_sched * ts)898 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
899 {
900 	/*
901 	 * If this CPU is offline and it is the one which updates
902 	 * jiffies, then give up the assignment and let it be taken by
903 	 * the CPU which runs the tick timer next. If we don't drop
904 	 * this here the jiffies might be stale and do_timer() never
905 	 * invoked.
906 	 */
907 	if (unlikely(!cpu_online(cpu))) {
908 		if (cpu == tick_do_timer_cpu)
909 			tick_do_timer_cpu = TICK_DO_TIMER_NONE;
910 		/*
911 		 * Make sure the CPU doesn't get fooled by obsolete tick
912 		 * deadline if it comes back online later.
913 		 */
914 		ts->next_tick = 0;
915 		return false;
916 	}
917 
918 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
919 		return false;
920 
921 	if (need_resched())
922 		return false;
923 
924 	if (unlikely(local_softirq_pending())) {
925 		static int ratelimit;
926 
927 		if (ratelimit < 10 &&
928 		    (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
929 			pr_warn("NOHZ tick-stop error: Non-RCU local softirq work is pending, handler #%02x!!!\n",
930 				(unsigned int) local_softirq_pending());
931 			ratelimit++;
932 		}
933 		return false;
934 	}
935 
936 	if (tick_nohz_full_enabled()) {
937 		/*
938 		 * Keep the tick alive to guarantee timekeeping progression
939 		 * if there are full dynticks CPUs around
940 		 */
941 		if (tick_do_timer_cpu == cpu)
942 			return false;
943 
944 		/* Should not happen for nohz-full */
945 		if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
946 			return false;
947 	}
948 
949 	return true;
950 }
951 
__tick_nohz_idle_stop_tick(struct tick_sched * ts)952 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
953 {
954 	ktime_t expires;
955 	int cpu = smp_processor_id();
956 
957 	/*
958 	 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
959 	 * tick timer expiration time is known already.
960 	 */
961 	if (ts->timer_expires_base)
962 		expires = ts->timer_expires;
963 	else if (can_stop_idle_tick(cpu, ts))
964 		expires = tick_nohz_next_event(ts, cpu);
965 	else
966 		return;
967 
968 	ts->idle_calls++;
969 
970 	if (expires > 0LL) {
971 		int was_stopped = ts->tick_stopped;
972 
973 		tick_nohz_stop_tick(ts, cpu);
974 
975 		ts->idle_sleeps++;
976 		ts->idle_expires = expires;
977 
978 		if (!was_stopped && ts->tick_stopped) {
979 			ts->idle_jiffies = ts->last_jiffies;
980 			nohz_balance_enter_idle(cpu);
981 		}
982 	} else {
983 		tick_nohz_retain_tick(ts);
984 	}
985 }
986 
987 /**
988  * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
989  *
990  * When the next event is more than a tick into the future, stop the idle tick
991  */
tick_nohz_idle_stop_tick(void)992 void tick_nohz_idle_stop_tick(void)
993 {
994 	__tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
995 }
996 
tick_nohz_idle_retain_tick(void)997 void tick_nohz_idle_retain_tick(void)
998 {
999 	tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
1000 	/*
1001 	 * Undo the effect of get_next_timer_interrupt() called from
1002 	 * tick_nohz_next_event().
1003 	 */
1004 	timer_clear_idle();
1005 }
1006 
1007 /**
1008  * tick_nohz_idle_enter - prepare for entering idle on the current CPU
1009  *
1010  * Called when we start the idle loop.
1011  */
tick_nohz_idle_enter(void)1012 void tick_nohz_idle_enter(void)
1013 {
1014 	struct tick_sched *ts;
1015 
1016 	lockdep_assert_irqs_enabled();
1017 
1018 	local_irq_disable();
1019 
1020 	ts = this_cpu_ptr(&tick_cpu_sched);
1021 
1022 	WARN_ON_ONCE(ts->timer_expires_base);
1023 
1024 	ts->inidle = 1;
1025 	tick_nohz_start_idle(ts);
1026 
1027 	local_irq_enable();
1028 }
1029 
1030 /**
1031  * tick_nohz_irq_exit - update next tick event from interrupt exit
1032  *
1033  * When an interrupt fires while we are idle and it doesn't cause
1034  * a reschedule, it may still add, modify or delete a timer, enqueue
1035  * an RCU callback, etc...
1036  * So we need to re-calculate and reprogram the next tick event.
1037  */
tick_nohz_irq_exit(void)1038 void tick_nohz_irq_exit(void)
1039 {
1040 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1041 
1042 	if (ts->inidle)
1043 		tick_nohz_start_idle(ts);
1044 	else
1045 		tick_nohz_full_update_tick(ts);
1046 }
1047 
1048 /**
1049  * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
1050  */
tick_nohz_idle_got_tick(void)1051 bool tick_nohz_idle_got_tick(void)
1052 {
1053 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1054 
1055 	if (ts->got_idle_tick) {
1056 		ts->got_idle_tick = 0;
1057 		return true;
1058 	}
1059 	return false;
1060 }
1061 
1062 /**
1063  * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
1064  * or the tick, whatever that expires first. Note that, if the tick has been
1065  * stopped, it returns the next hrtimer.
1066  *
1067  * Called from power state control code with interrupts disabled
1068  */
tick_nohz_get_next_hrtimer(void)1069 ktime_t tick_nohz_get_next_hrtimer(void)
1070 {
1071 	return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
1072 }
1073 
1074 /**
1075  * tick_nohz_get_sleep_length - return the expected length of the current sleep
1076  * @delta_next: duration until the next event if the tick cannot be stopped
1077  *
1078  * Called from power state control code with interrupts disabled
1079  */
tick_nohz_get_sleep_length(ktime_t * delta_next)1080 ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
1081 {
1082 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
1083 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1084 	int cpu = smp_processor_id();
1085 	/*
1086 	 * The idle entry time is expected to be a sufficient approximation of
1087 	 * the current time at this point.
1088 	 */
1089 	ktime_t now = ts->idle_entrytime;
1090 	ktime_t next_event;
1091 
1092 	WARN_ON_ONCE(!ts->inidle);
1093 
1094 	*delta_next = ktime_sub(dev->next_event, now);
1095 
1096 	if (!can_stop_idle_tick(cpu, ts))
1097 		return *delta_next;
1098 
1099 	next_event = tick_nohz_next_event(ts, cpu);
1100 	if (!next_event)
1101 		return *delta_next;
1102 
1103 	/*
1104 	 * If the next highres timer to expire is earlier than next_event, the
1105 	 * idle governor needs to know that.
1106 	 */
1107 	next_event = min_t(u64, next_event,
1108 			   hrtimer_next_event_without(&ts->sched_timer));
1109 
1110 	return ktime_sub(next_event, now);
1111 }
1112 
1113 /**
1114  * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1115  * for a particular CPU.
1116  *
1117  * Called from the schedutil frequency scaling governor in scheduler context.
1118  */
tick_nohz_get_idle_calls_cpu(int cpu)1119 unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
1120 {
1121 	struct tick_sched *ts = tick_get_tick_sched(cpu);
1122 
1123 	return ts->idle_calls;
1124 }
1125 
1126 /**
1127  * tick_nohz_get_idle_calls - return the current idle calls counter value
1128  *
1129  * Called from the schedutil frequency scaling governor in scheduler context.
1130  */
tick_nohz_get_idle_calls(void)1131 unsigned long tick_nohz_get_idle_calls(void)
1132 {
1133 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1134 
1135 	return ts->idle_calls;
1136 }
1137 
tick_nohz_account_idle_ticks(struct tick_sched * ts)1138 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
1139 {
1140 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1141 	unsigned long ticks;
1142 
1143 	if (vtime_accounting_enabled_this_cpu())
1144 		return;
1145 	/*
1146 	 * We stopped the tick in idle. Update process times would miss the
1147 	 * time we slept as update_process_times does only a 1 tick
1148 	 * accounting. Enforce that this is accounted to idle !
1149 	 */
1150 	ticks = jiffies - ts->idle_jiffies;
1151 	/*
1152 	 * We might be one off. Do not randomly account a huge number of ticks!
1153 	 */
1154 	if (ticks && ticks < LONG_MAX)
1155 		account_idle_ticks(ticks);
1156 #endif
1157 }
1158 
__tick_nohz_idle_restart_tick(struct tick_sched * ts,ktime_t now)1159 static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)
1160 {
1161 	tick_nohz_restart_sched_tick(ts, now);
1162 	tick_nohz_account_idle_ticks(ts);
1163 }
1164 
tick_nohz_idle_restart_tick(void)1165 void tick_nohz_idle_restart_tick(void)
1166 {
1167 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1168 
1169 	if (ts->tick_stopped)
1170 		__tick_nohz_idle_restart_tick(ts, ktime_get());
1171 }
1172 
1173 /**
1174  * tick_nohz_idle_exit - restart the idle tick from the idle task
1175  *
1176  * Restart the idle tick when the CPU is woken up from idle
1177  * This also exit the RCU extended quiescent state. The CPU
1178  * can use RCU again after this function is called.
1179  */
tick_nohz_idle_exit(void)1180 void tick_nohz_idle_exit(void)
1181 {
1182 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1183 	bool idle_active, tick_stopped;
1184 	ktime_t now;
1185 
1186 	local_irq_disable();
1187 
1188 	WARN_ON_ONCE(!ts->inidle);
1189 	WARN_ON_ONCE(ts->timer_expires_base);
1190 
1191 	ts->inidle = 0;
1192 	idle_active = ts->idle_active;
1193 	tick_stopped = ts->tick_stopped;
1194 
1195 	if (idle_active || tick_stopped)
1196 		now = ktime_get();
1197 
1198 	if (idle_active)
1199 		tick_nohz_stop_idle(ts, now);
1200 
1201 	if (tick_stopped)
1202 		__tick_nohz_idle_restart_tick(ts, now);
1203 
1204 	local_irq_enable();
1205 }
1206 
1207 /*
1208  * The nohz low res interrupt handler
1209  */
tick_nohz_handler(struct clock_event_device * dev)1210 static void tick_nohz_handler(struct clock_event_device *dev)
1211 {
1212 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1213 	struct pt_regs *regs = get_irq_regs();
1214 	ktime_t now = ktime_get();
1215 
1216 	dev->next_event = KTIME_MAX;
1217 
1218 	tick_sched_do_timer(ts, now);
1219 	tick_sched_handle(ts, regs);
1220 
1221 	/* No need to reprogram if we are running tickless  */
1222 	if (unlikely(ts->tick_stopped))
1223 		return;
1224 
1225 	hrtimer_forward(&ts->sched_timer, now, tick_period);
1226 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1227 }
1228 
tick_nohz_activate(struct tick_sched * ts,int mode)1229 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1230 {
1231 	if (!tick_nohz_enabled)
1232 		return;
1233 	ts->nohz_mode = mode;
1234 	/* One update is enough */
1235 	if (!test_and_set_bit(0, &tick_nohz_active))
1236 		timers_update_nohz();
1237 }
1238 
1239 /**
1240  * tick_nohz_switch_to_nohz - switch to nohz mode
1241  */
tick_nohz_switch_to_nohz(void)1242 static void tick_nohz_switch_to_nohz(void)
1243 {
1244 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1245 	ktime_t next;
1246 
1247 	if (!tick_nohz_enabled)
1248 		return;
1249 
1250 	if (tick_switch_to_oneshot(tick_nohz_handler))
1251 		return;
1252 
1253 	/*
1254 	 * Recycle the hrtimer in ts, so we can share the
1255 	 * hrtimer_forward with the highres code.
1256 	 */
1257 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1258 	/* Get the next period */
1259 	next = tick_init_jiffy_update();
1260 
1261 	hrtimer_set_expires(&ts->sched_timer, next);
1262 	hrtimer_forward_now(&ts->sched_timer, tick_period);
1263 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1264 	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1265 }
1266 
tick_nohz_irq_enter(void)1267 static inline void tick_nohz_irq_enter(void)
1268 {
1269 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1270 	ktime_t now;
1271 
1272 	if (!ts->idle_active && !ts->tick_stopped)
1273 		return;
1274 	now = ktime_get();
1275 	if (ts->idle_active)
1276 		tick_nohz_stop_idle(ts, now);
1277 	if (ts->tick_stopped)
1278 		tick_nohz_update_jiffies(now);
1279 }
1280 
1281 #else
1282 
tick_nohz_switch_to_nohz(void)1283 static inline void tick_nohz_switch_to_nohz(void) { }
tick_nohz_irq_enter(void)1284 static inline void tick_nohz_irq_enter(void) { }
tick_nohz_activate(struct tick_sched * ts,int mode)1285 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1286 
1287 #endif /* CONFIG_NO_HZ_COMMON */
1288 
1289 /*
1290  * Called from irq_enter to notify about the possible interruption of idle()
1291  */
tick_irq_enter(void)1292 void tick_irq_enter(void)
1293 {
1294 	tick_check_oneshot_broadcast_this_cpu();
1295 	tick_nohz_irq_enter();
1296 }
1297 
1298 /*
1299  * High resolution timer specific code
1300  */
1301 #ifdef CONFIG_HIGH_RES_TIMERS
1302 /*
1303  * We rearm the timer until we get disabled by the idle code.
1304  * Called with interrupts disabled.
1305  */
tick_sched_timer(struct hrtimer * timer)1306 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1307 {
1308 	struct tick_sched *ts =
1309 		container_of(timer, struct tick_sched, sched_timer);
1310 	struct pt_regs *regs = get_irq_regs();
1311 	ktime_t now = ktime_get();
1312 
1313 	tick_sched_do_timer(ts, now);
1314 
1315 	/*
1316 	 * Do not call, when we are not in irq context and have
1317 	 * no valid regs pointer
1318 	 */
1319 	if (regs)
1320 		tick_sched_handle(ts, regs);
1321 	else
1322 		ts->next_tick = 0;
1323 
1324 	/* No need to reprogram if we are in idle or full dynticks mode */
1325 	if (unlikely(ts->tick_stopped))
1326 		return HRTIMER_NORESTART;
1327 
1328 	hrtimer_forward(timer, now, tick_period);
1329 
1330 	return HRTIMER_RESTART;
1331 }
1332 
1333 static int sched_skew_tick;
1334 
skew_tick(char * str)1335 static int __init skew_tick(char *str)
1336 {
1337 	get_option(&str, &sched_skew_tick);
1338 
1339 	return 0;
1340 }
1341 early_param("skew_tick", skew_tick);
1342 
1343 /**
1344  * tick_setup_sched_timer - setup the tick emulation timer
1345  */
tick_setup_sched_timer(void)1346 void tick_setup_sched_timer(void)
1347 {
1348 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1349 	ktime_t now = ktime_get();
1350 
1351 	/*
1352 	 * Emulate tick processing via per-CPU hrtimers:
1353 	 */
1354 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1355 	ts->sched_timer.function = tick_sched_timer;
1356 
1357 	/* Get the next period (per-CPU) */
1358 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1359 
1360 	/* Offset the tick to avert jiffies_lock contention. */
1361 	if (sched_skew_tick) {
1362 		u64 offset = ktime_to_ns(tick_period) >> 1;
1363 		do_div(offset, num_possible_cpus());
1364 		offset *= smp_processor_id();
1365 		hrtimer_add_expires_ns(&ts->sched_timer, offset);
1366 	}
1367 
1368 	hrtimer_forward(&ts->sched_timer, now, tick_period);
1369 	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
1370 	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1371 }
1372 #endif /* HIGH_RES_TIMERS */
1373 
1374 #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
tick_cancel_sched_timer(int cpu)1375 void tick_cancel_sched_timer(int cpu)
1376 {
1377 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1378 
1379 # ifdef CONFIG_HIGH_RES_TIMERS
1380 	if (ts->sched_timer.base)
1381 		hrtimer_cancel(&ts->sched_timer);
1382 # endif
1383 
1384 	memset(ts, 0, sizeof(*ts));
1385 }
1386 #endif
1387 
1388 /**
1389  * Async notification about clocksource changes
1390  */
tick_clock_notify(void)1391 void tick_clock_notify(void)
1392 {
1393 	int cpu;
1394 
1395 	for_each_possible_cpu(cpu)
1396 		set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1397 }
1398 
1399 /*
1400  * Async notification about clock event changes
1401  */
tick_oneshot_notify(void)1402 void tick_oneshot_notify(void)
1403 {
1404 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1405 
1406 	set_bit(0, &ts->check_clocks);
1407 }
1408 
1409 /**
1410  * Check, if a change happened, which makes oneshot possible.
1411  *
1412  * Called cyclic from the hrtimer softirq (driven by the timer
1413  * softirq) allow_nohz signals, that we can switch into low-res nohz
1414  * mode, because high resolution timers are disabled (either compile
1415  * or runtime). Called with interrupts disabled.
1416  */
tick_check_oneshot_change(int allow_nohz)1417 int tick_check_oneshot_change(int allow_nohz)
1418 {
1419 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1420 
1421 	if (!test_and_clear_bit(0, &ts->check_clocks))
1422 		return 0;
1423 
1424 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1425 		return 0;
1426 
1427 	if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1428 		return 0;
1429 
1430 	if (!allow_nohz)
1431 		return 1;
1432 
1433 	tick_nohz_switch_to_nohz();
1434 	return 0;
1435 }
1436