1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Timer events oriented CPU idle governor
4 *
5 * TEO governor:
6 * Copyright (C) 2018 - 2021 Intel Corporation
7 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8 *
9 * Util-awareness mechanism:
10 * Copyright (C) 2022 Arm Ltd.
11 * Author: Kajetan Puchalski <kajetan.puchalski@arm.com>
12 */
13
14 /**
15 * DOC: teo-description
16 *
17 * The idea of this governor is based on the observation that on many systems
18 * timer events are two or more orders of magnitude more frequent than any
19 * other interrupts, so they are likely to be the most significant cause of CPU
20 * wakeups from idle states. Moreover, information about what happened in the
21 * (relatively recent) past can be used to estimate whether or not the deepest
22 * idle state with target residency within the (known) time till the closest
23 * timer event, referred to as the sleep length, is likely to be suitable for
24 * the upcoming CPU idle period and, if not, then which of the shallower idle
25 * states to choose instead of it.
26 *
27 * Of course, non-timer wakeup sources are more important in some use cases
28 * which can be covered by taking a few most recent idle time intervals of the
29 * CPU into account. However, even in that context it is not necessary to
30 * consider idle duration values greater than the sleep length, because the
31 * closest timer will ultimately wake up the CPU anyway unless it is woken up
32 * earlier.
33 *
34 * Thus this governor estimates whether or not the prospective idle duration of
35 * a CPU is likely to be significantly shorter than the sleep length and selects
36 * an idle state for it accordingly.
37 *
38 * The computations carried out by this governor are based on using bins whose
39 * boundaries are aligned with the target residency parameter values of the CPU
40 * idle states provided by the %CPUIdle driver in the ascending order. That is,
41 * the first bin spans from 0 up to, but not including, the target residency of
42 * the second idle state (idle state 1), the second bin spans from the target
43 * residency of idle state 1 up to, but not including, the target residency of
44 * idle state 2, the third bin spans from the target residency of idle state 2
45 * up to, but not including, the target residency of idle state 3 and so on.
46 * The last bin spans from the target residency of the deepest idle state
47 * supplied by the driver to infinity.
48 *
49 * Two metrics called "hits" and "intercepts" are associated with each bin.
50 * They are updated every time before selecting an idle state for the given CPU
51 * in accordance with what happened last time.
52 *
53 * The "hits" metric reflects the relative frequency of situations in which the
54 * sleep length and the idle duration measured after CPU wakeup fall into the
55 * same bin (that is, the CPU appears to wake up "on time" relative to the sleep
56 * length). In turn, the "intercepts" metric reflects the relative frequency of
57 * situations in which the measured idle duration is so much shorter than the
58 * sleep length that the bin it falls into corresponds to an idle state
59 * shallower than the one whose bin is fallen into by the sleep length (these
60 * situations are referred to as "intercepts" below).
61 *
62 * In addition to the metrics described above, the governor counts recent
63 * intercepts (that is, intercepts that have occurred during the last
64 * %NR_RECENT invocations of it for the given CPU) for each bin.
65 *
66 * In order to select an idle state for a CPU, the governor takes the following
67 * steps (modulo the possible latency constraint that must be taken into account
68 * too):
69 *
70 * 1. Find the deepest CPU idle state whose target residency does not exceed
71 * the current sleep length (the candidate idle state) and compute 3 sums as
72 * follows:
73 *
74 * - The sum of the "hits" and "intercepts" metrics for the candidate state
75 * and all of the deeper idle states (it represents the cases in which the
76 * CPU was idle long enough to avoid being intercepted if the sleep length
77 * had been equal to the current one).
78 *
79 * - The sum of the "intercepts" metrics for all of the idle states shallower
80 * than the candidate one (it represents the cases in which the CPU was not
81 * idle long enough to avoid being intercepted if the sleep length had been
82 * equal to the current one).
83 *
84 * - The sum of the numbers of recent intercepts for all of the idle states
85 * shallower than the candidate one.
86 *
87 * 2. If the second sum is greater than the first one or the third sum is
88 * greater than %NR_RECENT / 2, the CPU is likely to wake up early, so look
89 * for an alternative idle state to select.
90 *
91 * - Traverse the idle states shallower than the candidate one in the
92 * descending order.
93 *
94 * - For each of them compute the sum of the "intercepts" metrics and the sum
95 * of the numbers of recent intercepts over all of the idle states between
96 * it and the candidate one (including the former and excluding the
97 * latter).
98 *
99 * - If each of these sums that needs to be taken into account (because the
100 * check related to it has indicated that the CPU is likely to wake up
101 * early) is greater than a half of the corresponding sum computed in step
102 * 1 (which means that the target residency of the state in question had
103 * not exceeded the idle duration in over a half of the relevant cases),
104 * select the given idle state instead of the candidate one.
105 *
106 * 3. By default, select the candidate state.
107 *
108 * Util-awareness mechanism:
109 *
110 * The idea behind the util-awareness extension is that there are two distinct
111 * scenarios for the CPU which should result in two different approaches to idle
112 * state selection - utilized and not utilized.
113 *
114 * In this case, 'utilized' means that the average runqueue util of the CPU is
115 * above a certain threshold.
116 *
117 * When the CPU is utilized while going into idle, more likely than not it will
118 * be woken up to do more work soon and so a shallower idle state should be
119 * selected to minimise latency and maximise performance. When the CPU is not
120 * being utilized, the usual metrics-based approach to selecting the deepest
121 * available idle state should be preferred to take advantage of the power
122 * saving.
123 *
124 * In order to achieve this, the governor uses a utilization threshold.
125 * The threshold is computed per-CPU as a percentage of the CPU's capacity
126 * by bit shifting the capacity value. Based on testing, the shift of 6 (~1.56%)
127 * seems to be getting the best results.
128 *
129 * Before selecting the next idle state, the governor compares the current CPU
130 * util to the precomputed util threshold. If it's below, it defaults to the
131 * TEO metrics mechanism. If it's above, the closest shallower idle state will
132 * be selected instead, as long as is not a polling state.
133 */
134
135 #include <linux/cpuidle.h>
136 #include <linux/jiffies.h>
137 #include <linux/kernel.h>
138 #include <linux/sched.h>
139 #include <linux/sched/clock.h>
140 #include <linux/sched/topology.h>
141 #include <linux/tick.h>
142
143 #include "gov.h"
144
145 /*
146 * The number of bits to shift the CPU's capacity by in order to determine
147 * the utilized threshold.
148 *
149 * 6 was chosen based on testing as the number that achieved the best balance
150 * of power and performance on average.
151 *
152 * The resulting threshold is high enough to not be triggered by background
153 * noise and low enough to react quickly when activity starts to ramp up.
154 */
155 #define UTIL_THRESHOLD_SHIFT 6
156
157 /*
158 * The PULSE value is added to metrics when they grow and the DECAY_SHIFT value
159 * is used for decreasing metrics on a regular basis.
160 */
161 #define PULSE 1024
162 #define DECAY_SHIFT 3
163
164 /*
165 * Number of the most recent idle duration values to take into consideration for
166 * the detection of recent early wakeup patterns.
167 */
168 #define NR_RECENT 9
169
170 /**
171 * struct teo_bin - Metrics used by the TEO cpuidle governor.
172 * @intercepts: The "intercepts" metric.
173 * @hits: The "hits" metric.
174 * @recent: The number of recent "intercepts".
175 */
176 struct teo_bin {
177 unsigned int intercepts;
178 unsigned int hits;
179 unsigned int recent;
180 };
181
182 /**
183 * struct teo_cpu - CPU data used by the TEO cpuidle governor.
184 * @time_span_ns: Time between idle state selection and post-wakeup update.
185 * @sleep_length_ns: Time till the closest timer event (at the selection time).
186 * @state_bins: Idle state data bins for this CPU.
187 * @total: Grand total of the "intercepts" and "hits" metrics for all bins.
188 * @next_recent_idx: Index of the next @recent_idx entry to update.
189 * @recent_idx: Indices of bins corresponding to recent "intercepts".
190 * @tick_hits: Number of "hits" after TICK_NSEC.
191 * @util_threshold: Threshold above which the CPU is considered utilized
192 */
193 struct teo_cpu {
194 s64 time_span_ns;
195 s64 sleep_length_ns;
196 struct teo_bin state_bins[CPUIDLE_STATE_MAX];
197 unsigned int total;
198 int next_recent_idx;
199 int recent_idx[NR_RECENT];
200 unsigned int tick_hits;
201 unsigned long util_threshold;
202 };
203
204 static DEFINE_PER_CPU(struct teo_cpu, teo_cpus);
205
teo_cpu_get_util_threshold(int cpu)206 unsigned long teo_cpu_get_util_threshold(int cpu)
207 {
208 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, cpu);
209 return cpu_data->util_threshold;
210 }
211 EXPORT_SYMBOL_GPL(teo_cpu_get_util_threshold);
teo_cpu_set_util_threshold(int cpu,unsigned long util)212 void teo_cpu_set_util_threshold(int cpu, unsigned long util)
213 {
214 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, cpu);
215 cpu_data->util_threshold = util;
216 }
217 EXPORT_SYMBOL_GPL(teo_cpu_set_util_threshold);
218
219 /**
220 * teo_cpu_is_utilized - Check if the CPU's util is above the threshold
221 * @cpu: Target CPU
222 * @cpu_data: Governor CPU data for the target CPU
223 */
224 #ifdef CONFIG_SMP
teo_cpu_is_utilized(int cpu,struct teo_cpu * cpu_data)225 static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data)
226 {
227 return sched_cpu_util(cpu) > cpu_data->util_threshold;
228 }
229 #else
teo_cpu_is_utilized(int cpu,struct teo_cpu * cpu_data)230 static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data)
231 {
232 return false;
233 }
234 #endif
235
236 /**
237 * teo_update - Update CPU metrics after wakeup.
238 * @drv: cpuidle driver containing state data.
239 * @dev: Target CPU.
240 */
teo_update(struct cpuidle_driver * drv,struct cpuidle_device * dev)241 static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
242 {
243 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
244 int i, idx_timer = 0, idx_duration = 0;
245 s64 target_residency_ns;
246 u64 measured_ns;
247
248 if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {
249 /*
250 * One of the safety nets has triggered or the wakeup was close
251 * enough to the closest timer event expected at the idle state
252 * selection time to be discarded.
253 */
254 measured_ns = U64_MAX;
255 } else {
256 u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns;
257
258 /*
259 * The computations below are to determine whether or not the
260 * (saved) time till the next timer event and the measured idle
261 * duration fall into the same "bin", so use last_residency_ns
262 * for that instead of time_span_ns which includes the cpuidle
263 * overhead.
264 */
265 measured_ns = dev->last_residency_ns;
266 /*
267 * The delay between the wakeup and the first instruction
268 * executed by the CPU is not likely to be worst-case every
269 * time, so take 1/2 of the exit latency as a very rough
270 * approximation of the average of it.
271 */
272 if (measured_ns >= lat_ns)
273 measured_ns -= lat_ns / 2;
274 else
275 measured_ns /= 2;
276 }
277
278 cpu_data->total = 0;
279
280 /*
281 * Decay the "hits" and "intercepts" metrics for all of the bins and
282 * find the bins that the sleep length and the measured idle duration
283 * fall into.
284 */
285 for (i = 0; i < drv->state_count; i++) {
286 struct teo_bin *bin = &cpu_data->state_bins[i];
287
288 bin->hits -= bin->hits >> DECAY_SHIFT;
289 bin->intercepts -= bin->intercepts >> DECAY_SHIFT;
290
291 cpu_data->total += bin->hits + bin->intercepts;
292
293 target_residency_ns = drv->states[i].target_residency_ns;
294
295 if (target_residency_ns <= cpu_data->sleep_length_ns) {
296 idx_timer = i;
297 if (target_residency_ns <= measured_ns)
298 idx_duration = i;
299 }
300 }
301
302 i = cpu_data->next_recent_idx++;
303 if (cpu_data->next_recent_idx >= NR_RECENT)
304 cpu_data->next_recent_idx = 0;
305
306 if (cpu_data->recent_idx[i] >= 0)
307 cpu_data->state_bins[cpu_data->recent_idx[i]].recent--;
308
309 /*
310 * If the deepest state's target residency is below the tick length,
311 * make a record of it to help teo_select() decide whether or not
312 * to stop the tick. This effectively adds an extra hits-only bin
313 * beyond the last state-related one.
314 */
315 if (target_residency_ns < TICK_NSEC) {
316 cpu_data->tick_hits -= cpu_data->tick_hits >> DECAY_SHIFT;
317
318 cpu_data->total += cpu_data->tick_hits;
319
320 if (TICK_NSEC <= cpu_data->sleep_length_ns) {
321 idx_timer = drv->state_count;
322 if (TICK_NSEC <= measured_ns) {
323 cpu_data->tick_hits += PULSE;
324 goto end;
325 }
326 }
327 }
328
329 /*
330 * If the measured idle duration falls into the same bin as the sleep
331 * length, this is a "hit", so update the "hits" metric for that bin.
332 * Otherwise, update the "intercepts" metric for the bin fallen into by
333 * the measured idle duration.
334 */
335 if (idx_timer == idx_duration) {
336 cpu_data->state_bins[idx_timer].hits += PULSE;
337 cpu_data->recent_idx[i] = -1;
338 } else {
339 cpu_data->state_bins[idx_duration].intercepts += PULSE;
340 cpu_data->state_bins[idx_duration].recent++;
341 cpu_data->recent_idx[i] = idx_duration;
342 }
343
344 end:
345 cpu_data->total += PULSE;
346 }
347
teo_state_ok(int i,struct cpuidle_driver * drv)348 static bool teo_state_ok(int i, struct cpuidle_driver *drv)
349 {
350 return !tick_nohz_tick_stopped() ||
351 drv->states[i].target_residency_ns >= TICK_NSEC;
352 }
353
354 /**
355 * teo_find_shallower_state - Find shallower idle state matching given duration.
356 * @drv: cpuidle driver containing state data.
357 * @dev: Target CPU.
358 * @state_idx: Index of the capping idle state.
359 * @duration_ns: Idle duration value to match.
360 * @no_poll: Don't consider polling states.
361 */
teo_find_shallower_state(struct cpuidle_driver * drv,struct cpuidle_device * dev,int state_idx,s64 duration_ns,bool no_poll)362 static int teo_find_shallower_state(struct cpuidle_driver *drv,
363 struct cpuidle_device *dev, int state_idx,
364 s64 duration_ns, bool no_poll)
365 {
366 int i;
367
368 for (i = state_idx - 1; i >= 0; i--) {
369 if (dev->states_usage[i].disable ||
370 (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING))
371 continue;
372
373 state_idx = i;
374 if (drv->states[i].target_residency_ns <= duration_ns)
375 break;
376 }
377 return state_idx;
378 }
379
380 /**
381 * teo_select - Selects the next idle state to enter.
382 * @drv: cpuidle driver containing state data.
383 * @dev: Target CPU.
384 * @stop_tick: Indication on whether or not to stop the scheduler tick.
385 */
teo_select(struct cpuidle_driver * drv,struct cpuidle_device * dev,bool * stop_tick)386 static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
387 bool *stop_tick)
388 {
389 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
390 s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
391 ktime_t delta_tick = TICK_NSEC / 2;
392 unsigned int tick_intercept_sum = 0;
393 unsigned int idx_intercept_sum = 0;
394 unsigned int intercept_sum = 0;
395 unsigned int idx_recent_sum = 0;
396 unsigned int recent_sum = 0;
397 unsigned int idx_hit_sum = 0;
398 unsigned int hit_sum = 0;
399 int constraint_idx = 0;
400 int idx0 = 0, idx = -1;
401 bool alt_intercepts, alt_recent;
402 bool cpu_utilized;
403 s64 duration_ns;
404 int i;
405
406 if (dev->last_state_idx >= 0) {
407 teo_update(drv, dev);
408 dev->last_state_idx = -1;
409 }
410
411 cpu_data->time_span_ns = local_clock();
412 /*
413 * Set the expected sleep length to infinity in case of an early
414 * return.
415 */
416 cpu_data->sleep_length_ns = KTIME_MAX;
417
418 /* Check if there is any choice in the first place. */
419 if (drv->state_count < 2) {
420 idx = 0;
421 goto out_tick;
422 }
423
424 if (!dev->states_usage[0].disable)
425 idx = 0;
426
427 cpu_utilized = teo_cpu_is_utilized(dev->cpu, cpu_data);
428 /*
429 * If the CPU is being utilized over the threshold and there are only 2
430 * states to choose from, the metrics need not be considered, so choose
431 * the shallowest non-polling state and exit.
432 */
433 if (drv->state_count < 3 && cpu_utilized) {
434 /*
435 * If state 0 is enabled and it is not a polling one, select it
436 * right away unless the scheduler tick has been stopped, in
437 * which case care needs to be taken to leave the CPU in a deep
438 * enough state in case it is not woken up any time soon after
439 * all. If state 1 is disabled, though, state 0 must be used
440 * anyway.
441 */
442 if ((!idx && !(drv->states[0].flags & CPUIDLE_FLAG_POLLING) &&
443 teo_state_ok(0, drv)) || dev->states_usage[1].disable) {
444 idx = 0;
445 goto out_tick;
446 }
447 /* Assume that state 1 is not a polling one and use it. */
448 idx = 1;
449 duration_ns = drv->states[1].target_residency_ns;
450 goto end;
451 }
452
453 /* Compute the sums of metrics for early wakeup pattern detection. */
454 for (i = 1; i < drv->state_count; i++) {
455 struct teo_bin *prev_bin = &cpu_data->state_bins[i-1];
456 struct cpuidle_state *s = &drv->states[i];
457
458 /*
459 * Update the sums of idle state mertics for all of the states
460 * shallower than the current one.
461 */
462 intercept_sum += prev_bin->intercepts;
463 hit_sum += prev_bin->hits;
464 recent_sum += prev_bin->recent;
465
466 if (dev->states_usage[i].disable)
467 continue;
468
469 if (idx < 0)
470 idx0 = i; /* first enabled state */
471
472 idx = i;
473
474 if (s->exit_latency_ns <= latency_req)
475 constraint_idx = i;
476
477 /* Save the sums for the current state. */
478 idx_intercept_sum = intercept_sum;
479 idx_hit_sum = hit_sum;
480 idx_recent_sum = recent_sum;
481 }
482
483 /* Avoid unnecessary overhead. */
484 if (idx < 0) {
485 idx = 0; /* No states enabled, must use 0. */
486 goto out_tick;
487 }
488
489 if (idx == idx0) {
490 /*
491 * Only one idle state is enabled, so use it, but do not
492 * allow the tick to be stopped it is shallow enough.
493 */
494 duration_ns = drv->states[idx].target_residency_ns;
495 goto end;
496 }
497
498 tick_intercept_sum = intercept_sum +
499 cpu_data->state_bins[drv->state_count-1].intercepts;
500
501 /*
502 * If the sum of the intercepts metric for all of the idle states
503 * shallower than the current candidate one (idx) is greater than the
504 * sum of the intercepts and hits metrics for the candidate state and
505 * all of the deeper states, or the sum of the numbers of recent
506 * intercepts over all of the states shallower than the candidate one
507 * is greater than a half of the number of recent events taken into
508 * account, a shallower idle state is likely to be a better choice.
509 */
510 alt_intercepts = 2 * idx_intercept_sum > cpu_data->total - idx_hit_sum;
511 alt_recent = idx_recent_sum > NR_RECENT / 2;
512 if (alt_recent || alt_intercepts) {
513 int first_suitable_idx = idx;
514
515 /*
516 * Look for the deepest idle state whose target residency had
517 * not exceeded the idle duration in over a half of the relevant
518 * cases (both with respect to intercepts overall and with
519 * respect to the recent intercepts only) in the past.
520 *
521 * Take the possible duration limitation present if the tick
522 * has been stopped already into account.
523 */
524 intercept_sum = 0;
525 recent_sum = 0;
526
527 for (i = idx - 1; i >= 0; i--) {
528 struct teo_bin *bin = &cpu_data->state_bins[i];
529
530 intercept_sum += bin->intercepts;
531 recent_sum += bin->recent;
532
533 if ((!alt_recent || 2 * recent_sum > idx_recent_sum) &&
534 (!alt_intercepts ||
535 2 * intercept_sum > idx_intercept_sum)) {
536 /*
537 * Use the current state unless it is too
538 * shallow or disabled, in which case take the
539 * first enabled state that is deep enough.
540 */
541 if (teo_state_ok(i, drv) &&
542 !dev->states_usage[i].disable)
543 idx = i;
544 else
545 idx = first_suitable_idx;
546
547 break;
548 }
549
550 if (dev->states_usage[i].disable)
551 continue;
552
553 if (!teo_state_ok(i, drv)) {
554 /*
555 * The current state is too shallow, but if an
556 * alternative candidate state has been found,
557 * it may still turn out to be a better choice.
558 */
559 if (first_suitable_idx != idx)
560 continue;
561
562 break;
563 }
564
565 first_suitable_idx = i;
566 }
567 }
568
569 /*
570 * If there is a latency constraint, it may be necessary to select an
571 * idle state shallower than the current candidate one.
572 */
573 if (idx > constraint_idx)
574 idx = constraint_idx;
575
576 /*
577 * If the CPU is being utilized over the threshold, choose a shallower
578 * non-polling state to improve latency, unless the scheduler tick has
579 * been stopped already and the shallower state's target residency is
580 * not sufficiently large.
581 */
582 if (cpu_utilized) {
583 i = teo_find_shallower_state(drv, dev, idx, KTIME_MAX, true);
584 if (teo_state_ok(i, drv))
585 idx = i;
586 }
587
588 /*
589 * Skip the timers check if state 0 is the current candidate one,
590 * because an immediate non-timer wakeup is expected in that case.
591 */
592 if (!idx)
593 goto out_tick;
594
595 /*
596 * If state 0 is a polling one, check if the target residency of
597 * the current candidate state is low enough and skip the timers
598 * check in that case too.
599 */
600 if ((drv->states[0].flags & CPUIDLE_FLAG_POLLING) &&
601 drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS)
602 goto out_tick;
603
604 duration_ns = tick_nohz_get_sleep_length(&delta_tick);
605 cpu_data->sleep_length_ns = duration_ns;
606
607 /*
608 * If the closest expected timer is before the terget residency of the
609 * candidate state, a shallower one needs to be found.
610 */
611 if (drv->states[idx].target_residency_ns > duration_ns) {
612 i = teo_find_shallower_state(drv, dev, idx, duration_ns, false);
613 if (teo_state_ok(i, drv))
614 idx = i;
615 }
616
617 /*
618 * If the selected state's target residency is below the tick length
619 * and intercepts occurring before the tick length are the majority of
620 * total wakeup events, do not stop the tick.
621 */
622 if (drv->states[idx].target_residency_ns < TICK_NSEC &&
623 tick_intercept_sum > cpu_data->total / 2 + cpu_data->total / 8)
624 duration_ns = TICK_NSEC / 2;
625
626 end:
627 /*
628 * Allow the tick to be stopped unless the selected state is a polling
629 * one or the expected idle duration is shorter than the tick period
630 * length.
631 */
632 if ((!(drv->states[idx].flags & CPUIDLE_FLAG_POLLING) &&
633 duration_ns >= TICK_NSEC) || tick_nohz_tick_stopped())
634 return idx;
635
636 /*
637 * The tick is not going to be stopped, so if the target residency of
638 * the state to be returned is not within the time till the closest
639 * timer including the tick, try to correct that.
640 */
641 if (idx > idx0 &&
642 drv->states[idx].target_residency_ns > delta_tick)
643 idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false);
644
645 out_tick:
646 *stop_tick = false;
647 return idx;
648 }
649
650 /**
651 * teo_reflect - Note that governor data for the CPU need to be updated.
652 * @dev: Target CPU.
653 * @state: Entered state.
654 */
teo_reflect(struct cpuidle_device * dev,int state)655 static void teo_reflect(struct cpuidle_device *dev, int state)
656 {
657 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
658
659 dev->last_state_idx = state;
660 /*
661 * If the wakeup was not "natural", but triggered by one of the safety
662 * nets, assume that the CPU might have been idle for the entire sleep
663 * length time.
664 */
665 if (dev->poll_time_limit ||
666 (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) {
667 dev->poll_time_limit = false;
668 cpu_data->time_span_ns = cpu_data->sleep_length_ns;
669 } else {
670 cpu_data->time_span_ns = local_clock() - cpu_data->time_span_ns;
671 }
672 }
673
674 /**
675 * teo_enable_device - Initialize the governor's data for the target CPU.
676 * @drv: cpuidle driver (not used).
677 * @dev: Target CPU.
678 */
teo_enable_device(struct cpuidle_driver * drv,struct cpuidle_device * dev)679 static int teo_enable_device(struct cpuidle_driver *drv,
680 struct cpuidle_device *dev)
681 {
682 struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
683 unsigned long max_capacity = arch_scale_cpu_capacity(dev->cpu);
684 int i;
685
686 memset(cpu_data, 0, sizeof(*cpu_data));
687 cpu_data->util_threshold = max_capacity >> UTIL_THRESHOLD_SHIFT;
688
689 for (i = 0; i < NR_RECENT; i++)
690 cpu_data->recent_idx[i] = -1;
691
692 return 0;
693 }
694
695 static struct cpuidle_governor teo_governor = {
696 .name = "teo",
697 .rating = 19,
698 .enable = teo_enable_device,
699 .select = teo_select,
700 .reflect = teo_reflect,
701 };
702
teo_governor_init(void)703 static int __init teo_governor_init(void)
704 {
705 return cpuidle_register_governor(&teo_governor);
706 }
707
708 postcore_initcall(teo_governor_init);
709