7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
13 static inline u64 thermal_load_avg(struct rq *rq)  in thermal_load_avg()  argument
15 	return READ_ONCE(rq->avg_thermal.load_avg);  in thermal_load_avg()
19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)  in update_thermal_load_avg()  argument
24 static inline u64 thermal_load_avg(struct rq *rq)  in thermal_load_avg()  argument
31 int update_irq_load_avg(struct rq *rq, u64 running);
34 update_irq_load_avg(struct rq *rq, u64 running)  in update_irq_load_avg()  argument
64 static inline u64 rq_clock_task_mult(struct rq *rq)  in rq_clock_task_mult()  argument
66 	lockdep_assert_rq_held(rq);  in rq_clock_task_mult()
67 	assert_clock_updated(rq);  in rq_clock_task_mult()
69 	return rq->clock_task_mult;  in rq_clock_task_mult()
72 static inline u64 rq_clock_pelt(struct rq *rq)  in rq_clock_pelt()  argument
74 	lockdep_assert_rq_held(rq);  in rq_clock_pelt()
75 	assert_clock_updated(rq);  in rq_clock_pelt()
77 	return rq->clock_pelt - rq->lost_idle_time;  in rq_clock_pelt()
81 static inline void _update_idle_rq_clock_pelt(struct rq *rq)  in _update_idle_rq_clock_pelt()  argument
83 	rq->clock_pelt = rq_clock_task_mult(rq);  in _update_idle_rq_clock_pelt()
85 	u64_u32_store(rq->clock_idle, rq_clock(rq));  in _update_idle_rq_clock_pelt()
88 	u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));  in _update_idle_rq_clock_pelt()
103 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)  in update_rq_clock_pelt()  argument
105 	if (unlikely(is_idle_task(rq->curr))) {  in update_rq_clock_pelt()
106 		_update_idle_rq_clock_pelt(rq);  in update_rq_clock_pelt()
126 	delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));  in update_rq_clock_pelt()
127 	delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));  in update_rq_clock_pelt()
129 	rq->clock_pelt += delta;  in update_rq_clock_pelt()
144 static inline void update_rq_clock_task_mult(struct rq *rq, s64 delta)  in update_rq_clock_task_mult()  argument
148 	rq->clock_task_mult += delta;  in update_rq_clock_task_mult()
150 	update_rq_clock_pelt(rq, delta);  in update_rq_clock_task_mult()
162 static inline void update_idle_rq_clock_pelt(struct rq *rq)  in update_idle_rq_clock_pelt()  argument
165 	u32 util_sum = rq->cfs.avg.util_sum;  in update_idle_rq_clock_pelt()
166 	util_sum += rq->avg_rt.util_sum;  in update_idle_rq_clock_pelt()
167 	util_sum += rq->avg_dl.util_sum;  in update_idle_rq_clock_pelt()
179 		rq->lost_idle_time += rq_clock_task_mult(rq) - rq->clock_pelt;  in update_idle_rq_clock_pelt()
181 	_update_idle_rq_clock_pelt(rq);  in update_idle_rq_clock_pelt()
222 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)  in update_rt_rq_load_avg()  argument
228 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)  in update_dl_rq_load_avg()  argument
234 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)  in update_thermal_load_avg()  argument
239 static inline u64 thermal_load_avg(struct rq *rq)  in thermal_load_avg()  argument
245 update_irq_load_avg(struct rq *rq, u64 running)  in update_irq_load_avg()  argument
250 static inline u64 rq_clock_task_mult(struct rq *rq)  in rq_clock_task_mult()  argument
252 	return rq_clock_task(rq);  in rq_clock_task_mult()
255 static inline u64 rq_clock_pelt(struct rq *rq)  in rq_clock_pelt()  argument
257 	return rq_clock_task_mult(rq);  in rq_clock_pelt()
261 update_rq_clock_task_mult(struct rq *rq, s64 delta) { }  in update_rq_clock_task_mult()  argument
264 update_idle_rq_clock_pelt(struct rq *rq) { }  in update_idle_rq_clock_pelt()  argument