Lines Matching +full:boost +full:- +full:max +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0
3 * CPUFreq governor based on scheduler-provided CPU utilization data.
58 /* The field below is for single-CPU policies only: */
73 * Since cpufreq_update_util() is called with rq->lock held for in sugov_should_update_freq()
74 * the @target_cpu, our per-CPU data is fully serialized. in sugov_should_update_freq()
76 * However, drivers cannot in general deal with cross-CPU in sugov_should_update_freq()
87 if (!cpufreq_this_cpu_can_update(sg_policy->policy)) in sugov_should_update_freq()
90 if (unlikely(sg_policy->limits_changed)) { in sugov_should_update_freq()
91 sg_policy->limits_changed = false; in sugov_should_update_freq()
92 sg_policy->need_freq_update = true; in sugov_should_update_freq()
96 delta_ns = time - sg_policy->last_freq_update_time; in sugov_should_update_freq()
98 return delta_ns >= sg_policy->freq_update_delay_ns; in sugov_should_update_freq()
104 if (sg_policy->need_freq_update) { in sugov_update_next_freq()
105 sg_policy->need_freq_update = false; in sugov_update_next_freq()
114 if (sg_policy->next_freq == next_freq && in sugov_update_next_freq()
117 } else if (sg_policy->next_freq == next_freq) { in sugov_update_next_freq()
121 sg_policy->next_freq = next_freq; in sugov_update_next_freq()
122 sg_policy->last_freq_update_time = time; in sugov_update_next_freq()
129 if (!sg_policy->work_in_progress) { in sugov_deferred_update()
130 sg_policy->work_in_progress = true; in sugov_deferred_update()
131 irq_work_queue(&sg_policy->irq_work); in sugov_deferred_update()
136 * get_next_freq - Compute a new frequency for a given cpufreq policy.
138 * @util: Current CPU utilization.
139 * @max: CPU capacity.
141 * If the utilization is frequency-invariant, choose the new frequency to be
144 * next_freq = C * max_freq * util / max
146 * Otherwise, approximate the would-be frequency-invariant utilization by
149 * next_freq = C * curr_freq * util_raw / max
151 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
153 * The lowest driver-supported frequency which is equal or greater than the raw
154 * next_freq (as calculated above) is returned, subject to policy min/max and
158 unsigned long util, unsigned long max) in get_next_freq() argument
160 struct cpufreq_policy *policy = sg_policy->policy; in get_next_freq()
162 policy->cpuinfo.max_freq : policy->cur; in get_next_freq()
165 freq = map_util_freq(util, freq, max); in get_next_freq()
167 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) in get_next_freq()
168 return sg_policy->next_freq; in get_next_freq()
170 sg_policy->cached_raw_freq = freq; in get_next_freq()
176 unsigned long util = cpu_util_cfs_boost(sg_cpu->cpu); in sugov_get_util()
177 struct rq *rq = cpu_rq(sg_cpu->cpu); in sugov_get_util()
179 sg_cpu->bw_dl = cpu_bw_dl(rq); in sugov_get_util()
181 cpu_util_freq_walt(sg_cpu->cpu); in sugov_get_util()
183 sg_cpu->util = effective_cpu_util(sg_cpu->cpu, util, in sugov_get_util()
189 * sugov_iowait_reset() - Reset the IO boost status of a CPU.
190 * @sg_cpu: the sugov data for the CPU to boost
192 * @set_iowait_boost: true if an IO boost has been requested
194 * The IO wait boost of a task is disabled after a tick since the last update
195 * of a CPU. If a new IO wait boost is requested after more then a tick, then
196 * we enable the boost starting from IOWAIT_BOOST_MIN, which improves energy
202 s64 delta_ns = time - sg_cpu->last_update; in sugov_iowait_reset()
204 /* Reset boost only if a tick has elapsed since last request */ in sugov_iowait_reset()
208 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; in sugov_iowait_reset()
209 sg_cpu->iowait_boost_pending = set_iowait_boost; in sugov_iowait_reset()
215 * sugov_iowait_boost() - Updates the IO boost status of a CPU.
216 * @sg_cpu: the sugov data for the CPU to boost
225 * To keep doubling, an IO boost has to be requested at least once per tick,
233 /* Reset boost if the CPU appears to have been idle enough */ in sugov_iowait_boost()
234 if (sg_cpu->iowait_boost && in sugov_iowait_boost()
238 /* Boost only tasks waking up after IO */ in sugov_iowait_boost()
242 /* Ensure boost doubles only one time at each request */ in sugov_iowait_boost()
243 if (sg_cpu->iowait_boost_pending) in sugov_iowait_boost()
245 sg_cpu->iowait_boost_pending = true; in sugov_iowait_boost()
247 /* Double the boost at each request */ in sugov_iowait_boost()
248 if (sg_cpu->iowait_boost) { in sugov_iowait_boost()
249 sg_cpu->iowait_boost = in sugov_iowait_boost()
250 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); in sugov_iowait_boost()
254 /* First wakeup after IO: start with minimum boost */ in sugov_iowait_boost()
255 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; in sugov_iowait_boost()
259 * sugov_iowait_apply() - Apply the IO boost to a CPU.
260 * @sg_cpu: the sugov data for the cpu to boost
262 * @max_cap: the max CPU capacity
266 * The IO boost value is increased each time a task wakes up from IO, in
271 * its IO boost utilization reset.
273 * This mechanism is designed to boost high frequently IO waiting tasks, while
279 unsigned long boost; in sugov_iowait_apply() local
281 /* No boost currently required */ in sugov_iowait_apply()
282 if (!sg_cpu->iowait_boost) in sugov_iowait_apply()
285 /* Reset boost if the CPU appears to have been idle enough */ in sugov_iowait_apply()
289 if (!sg_cpu->iowait_boost_pending) { in sugov_iowait_apply()
291 * No boost pending; reduce the boost value. in sugov_iowait_apply()
293 sg_cpu->iowait_boost >>= 1; in sugov_iowait_apply()
294 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { in sugov_iowait_apply()
295 sg_cpu->iowait_boost = 0; in sugov_iowait_apply()
300 sg_cpu->iowait_boost_pending = false; in sugov_iowait_apply()
303 * sg_cpu->util is already in capacity scale; convert iowait_boost in sugov_iowait_apply()
306 boost = (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT; in sugov_iowait_apply()
307 boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL); in sugov_iowait_apply()
308 if (sg_cpu->util < boost) in sugov_iowait_apply()
309 sg_cpu->util = boost; in sugov_iowait_apply()
315 unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); in sugov_cpu_is_busy()
316 bool ret = idle_calls == sg_cpu->saved_idle_calls; in sugov_cpu_is_busy()
318 sg_cpu->saved_idle_calls = idle_calls; in sugov_cpu_is_busy()
331 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl) in ignore_dl_rate_limit()
332 sg_cpu->sg_policy->limits_changed = true; in ignore_dl_rate_limit()
340 sg_cpu->last_update = time; in sugov_update_single_common()
344 if (!sugov_should_update_freq(sg_cpu->sg_policy, time)) in sugov_update_single_common()
357 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_single_freq()
358 unsigned int cached_freq = sg_policy->cached_raw_freq; in sugov_update_single_freq()
370 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_freq()
375 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap); in sugov_update_single_freq()
382 if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) && in sugov_update_single_freq()
383 sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq && in sugov_update_single_freq()
384 !sg_policy->need_freq_update) { in sugov_update_single_freq()
385 next_f = sg_policy->next_freq; in sugov_update_single_freq()
388 sg_policy->cached_raw_freq = cached_freq; in sugov_update_single_freq()
395 * This code runs under rq->lock for the target CPU, so it won't run in sugov_update_single_freq()
399 if (sg_policy->policy->fast_switch_enabled) { in sugov_update_single_freq()
400 cpufreq_driver_fast_switch(sg_policy->policy, next_f); in sugov_update_single_freq()
403 raw_spin_lock_irqsave(&sg_policy->update_lock, irq_flag); in sugov_update_single_freq()
405 raw_spin_lock(&sg_policy->update_lock); in sugov_update_single_freq()
409 raw_spin_unlock_irqrestore(&sg_policy->update_lock, irq_flag); in sugov_update_single_freq()
411 raw_spin_unlock(&sg_policy->update_lock); in sugov_update_single_freq()
420 unsigned long prev_util = sg_cpu->util; in sugov_update_single_perf()
433 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_perf()
444 if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) && in sugov_update_single_perf()
445 sugov_cpu_is_busy(sg_cpu) && sg_cpu->util < prev_util) in sugov_update_single_perf()
446 sg_cpu->util = prev_util; in sugov_update_single_perf()
448 cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl), in sugov_update_single_perf()
449 map_util_perf(sg_cpu->util), max_cap); in sugov_update_single_perf()
451 sg_cpu->sg_policy->last_freq_update_time = time; in sugov_update_single_perf()
456 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_next_freq_shared()
457 struct cpufreq_policy *policy = sg_policy->policy; in sugov_next_freq_shared()
461 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_next_freq_shared()
463 for_each_cpu(j, policy->cpus) { in sugov_next_freq_shared()
469 util = max(j_sg_cpu->util, util); in sugov_next_freq_shared()
473 sched_get_max_group_util(policy->cpus, &sg_policy->rtg_util, &sg_policy->rtg_freq); in sugov_next_freq_shared()
474 util = max(sg_policy->rtg_util, util); in sugov_next_freq_shared()
483 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
492 raw_spin_lock_irqsave(&sg_policy->update_lock, irq_flag);
494 raw_spin_lock(&sg_policy->update_lock);
498 sg_cpu->last_update = time;
513 if (sg_policy->policy->fast_switch_enabled)
514 cpufreq_driver_fast_switch(sg_policy->policy, next_f);
520 raw_spin_unlock_irqrestore(&sg_policy->update_lock, irq_flag);
522 raw_spin_unlock(&sg_policy->update_lock);
533 * Hold sg_policy->update_lock shortly to handle the case where:
534 * in case sg_policy->next_freq is read here, and then updated by
542 raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
543 freq = sg_policy->next_freq;
544 sg_policy->work_in_progress = false;
545 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
547 mutex_lock(&sg_policy->work_lock);
548 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);
549 mutex_unlock(&sg_policy->work_lock);
558 kthread_queue_work(&sg_policy->worker, &sg_policy->work);
575 return sprintf(buf, "%u\n", tunables->rate_limit_us);
586 return -EINVAL;
588 tunables->rate_limit_us = rate_limit_us;
590 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
591 sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;
629 sg_policy->policy = policy;
630 raw_spin_lock_init(&sg_policy->update_lock);
656 struct cpufreq_policy *policy = sg_policy->policy;
660 if (policy->fast_switch_enabled)
663 kthread_init_work(&sg_policy->work, sugov_work);
664 kthread_init_worker(&sg_policy->worker);
665 thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
667 cpumask_first(policy->related_cpus));
680 sg_policy->thread = thread;
681 kthread_bind_mask(thread, policy->related_cpus);
682 init_irq_work(&sg_policy->irq_work, sugov_irq_work);
683 mutex_init(&sg_policy->work_lock);
693 if (sg_policy->policy->fast_switch_enabled)
696 kthread_flush_worker(&sg_policy->worker);
697 kthread_stop(sg_policy->thread);
698 mutex_destroy(&sg_policy->work_lock);
707 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
727 if (policy->governor_data)
728 return -EBUSY;
734 ret = -ENOMEM;
746 ret = -EINVAL;
749 policy->governor_data = sg_policy;
750 sg_policy->tunables = global_tunables;
752 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
758 ret = -ENOMEM;
762 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
764 policy->governor_data = sg_policy;
765 sg_policy->tunables = tunables;
767 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
778 kobject_put(&tunables->attr_set.kobj);
779 policy->governor_data = NULL;
798 struct sugov_policy *sg_policy = policy->governor_data;
799 struct sugov_tunables *tunables = sg_policy->tunables;
804 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
805 policy->governor_data = NULL;
818 struct sugov_policy *sg_policy = policy->governor_data;
822 sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
823 sg_policy->last_freq_update_time = 0;
824 sg_policy->next_freq = 0;
825 sg_policy->work_in_progress = false;
826 sg_policy->limits_changed = false;
827 sg_policy->cached_raw_freq = 0;
829 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);
831 for_each_cpu(cpu, policy->cpus) {
835 sg_cpu->cpu = cpu;
836 sg_cpu->sg_policy = sg_policy;
841 else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())
846 for_each_cpu(cpu, policy->cpus) {
849 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);
856 struct sugov_policy *sg_policy = policy->governor_data;
859 for_each_cpu(cpu, policy->cpus)
864 if (!policy->fast_switch_enabled) {
865 irq_work_sync(&sg_policy->irq_work);
866 kthread_cancel_work_sync(&sg_policy->work);
872 struct sugov_policy *sg_policy = policy->governor_data;
874 if (!policy->fast_switch_enabled) {
875 mutex_lock(&sg_policy->work_lock);
877 mutex_unlock(&sg_policy->work_lock);
880 sg_policy->limits_changed = true;
917 if (old_gov == &schedutil_gov || policy->governor == &schedutil_gov) {