1 // SPDX-License-Identifier: GPL-2.0
17  * Of course, non-timer wakeup sources are more important in some use cases and
28  * - Find an idle state on the basis of the sleep length and state statistics
42  * - If the majority of the most recent idle duration values are below the
63  * the detection of wakeup patterns.
68  * struct teo_idle_state - Idle state data used by the TEO cpuidle governor.
73  * A CPU wakeup is "matched" by a given idle state if the idle duration measured
74  * after the wakeup is between the target residency of that state and the target
76  * "matches" a CPU wakeup when the measured idle duration is at least equal to
79  * Also, from the TEO governor perspective, a CPU wakeup from idle is "early" if
82  * time till the closest timer event).  Otherwise, the wakeup is "on time", or
85  * A "miss" occurs when the given state doesn't match the wakeup, but it matches
95  * struct teo_cpu - CPU data used by the TEO cpuidle governor.
96  * @time_span_ns: Time between idle state selection and post-wakeup update.
113  * teo_update - Update CPU data after wakeup.
119 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_update()
120 	int i, idx_hit = -1, idx_timer = -1;  in teo_update()
123 	if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {  in teo_update()
125 		 * One of the safety nets has triggered or the wakeup was close  in teo_update()
131 		u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns;  in teo_update()
140 		measured_ns = dev->last_residency_ns;  in teo_update()
142 		 * The delay between the wakeup and the first instruction  in teo_update()
143 		 * executed by the CPU is not likely to be worst-case every  in teo_update()
144 		 * time, so take 1/2 of the exit latency as a very rough  in teo_update()
148 			measured_ns -= lat_ns / 2;  in teo_update()
157 	for (i = 0; i < drv->state_count; i++) {  in teo_update()
158 		unsigned int early_hits = cpu_data->states[i].early_hits;  in teo_update()
160 		cpu_data->states[i].early_hits -= early_hits >> DECAY_SHIFT;  in teo_update()
162 		if (drv->states[i].target_residency_ns <= cpu_data->sleep_length_ns) {  in teo_update()
164 			if (drv->states[i].target_residency_ns <= measured_ns)  in teo_update()
178 		unsigned int hits = cpu_data->states[idx_timer].hits;  in teo_update()
179 		unsigned int misses = cpu_data->states[idx_timer].misses;  in teo_update()
181 		hits -= hits >> DECAY_SHIFT;  in teo_update()
182 		misses -= misses >> DECAY_SHIFT;  in teo_update()
187 				cpu_data->states[idx_hit].early_hits += PULSE;  in teo_update()
192 		cpu_data->states[idx_timer].misses = misses;  in teo_update()
193 		cpu_data->states[idx_timer].hits = hits;  in teo_update()
197 	 * Save idle duration values corresponding to non-timer wakeups for  in teo_update()
200 	cpu_data->intervals[cpu_data->interval_idx++] = measured_ns;  in teo_update()
201 	if (cpu_data->interval_idx >= INTERVALS)  in teo_update()
202 		cpu_data->interval_idx = 0;  in teo_update()
211  * teo_find_shallower_state - Find shallower idle state matching given duration.
223 	for (i = state_idx - 1; i >= 0; i--) {  in teo_find_shallower_state()
224 		if (dev->states_usage[i].disable)  in teo_find_shallower_state()
228 		if (drv->states[i].target_residency_ns <= duration_ns)  in teo_find_shallower_state()
235  * teo_select - Selects the next idle state to enter.
243 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_select()
244 	s64 latency_req = cpuidle_governor_latency_req(dev->cpu);  in teo_select()
250 	if (dev->last_state_idx >= 0) {  in teo_select()
252 		dev->last_state_idx = -1;  in teo_select()
255 	cpu_data->time_span_ns = local_clock();  in teo_select()
258 	cpu_data->sleep_length_ns = duration_ns;  in teo_select()
263 	max_early_idx = -1;  in teo_select()
264 	prev_max_early_idx = -1;  in teo_select()
265 	constraint_idx = drv->state_count;  in teo_select()
266 	idx = -1;  in teo_select()
268 	for (i = 0; i < drv->state_count; i++) {  in teo_select()
269 		struct cpuidle_state *s = &drv->states[i];  in teo_select()
271 		if (dev->states_usage[i].disable) {  in teo_select()
276 			if (s->target_residency_ns > duration_ns)  in teo_select()
287 			hits = cpu_data->states[i].hits;  in teo_select()
288 			misses = cpu_data->states[i].misses;  in teo_select()
290 			if (early_hits >= cpu_data->states[i].early_hits ||  in teo_select()
302 				early_hits = cpu_data->states[i].early_hits;  in teo_select()
314 			if (teo_time_ok(drv->states[idx].target_residency_ns)) {  in teo_select()
316 				early_hits = cpu_data->states[i].early_hits;  in teo_select()
325 			hits = cpu_data->states[i].hits;  in teo_select()
326 			misses = cpu_data->states[i].misses;  in teo_select()
329 		if (s->target_residency_ns > duration_ns)  in teo_select()
332 		if (s->exit_latency_ns > latency_req && constraint_idx > i)  in teo_select()
336 		hits = cpu_data->states[i].hits;  in teo_select()
337 		misses = cpu_data->states[i].misses;  in teo_select()
339 		if (early_hits < cpu_data->states[i].early_hits &&  in teo_select()
340 		    teo_time_ok(drv->states[i].target_residency_ns)) {  in teo_select()
342 			early_hits = cpu_data->states[i].early_hits;  in teo_select()
351 	 * idle duration observed after wakeup, so take the one with the maximum  in teo_select()
366 			duration_ns = drv->states[idx].target_residency_ns;  in teo_select()
371 	 * If there is a latency constraint, it may be necessary to use a  in teo_select()
388 			u64 val = cpu_data->intervals[i];  in teo_select()
410 				if (drv->states[idx].target_residency_ns > avg_ns)  in teo_select()
421 	if (((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) ||  in teo_select()
431 		if (idx > 0 && drv->states[idx].target_residency_ns > delta_tick)  in teo_select()
439  * teo_reflect - Note that governor data for the CPU need to be updated.
445 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_reflect()
447 	dev->last_state_idx = state;  in teo_reflect()
449 	 * If the wakeup was not "natural", but triggered by one of the safety  in teo_reflect()
453 	if (dev->poll_time_limit ||  in teo_reflect()
454 	    (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) {  in teo_reflect()
455 		dev->poll_time_limit = false;  in teo_reflect()
456 		cpu_data->time_span_ns = cpu_data->sleep_length_ns;  in teo_reflect()
458 		cpu_data->time_span_ns = local_clock() - cpu_data->time_span_ns;  in teo_reflect()
463  * teo_enable_device - Initialize the governor's data for the target CPU.
470 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_enable_device()
476 		cpu_data->intervals[i] = U64_MAX;  in teo_enable_device()