58 	struct rq *rq;  in sched_core_update_cookie()  local
60 	rq = task_rq_lock(p, &rf);  in sched_core_update_cookie()
68 	SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));  in sched_core_update_cookie()
71 		sched_core_dequeue(rq, p, DEQUEUE_SAVE);  in sched_core_update_cookie()
80 		sched_core_enqueue(rq, p);  in sched_core_update_cookie()
91 	if (task_on_cpu(rq, p))  in sched_core_update_cookie()
92 		resched_curr(rq);  in sched_core_update_cookie()
94 	task_rq_unlock(rq, p, &rf);  in sched_core_update_cookie()
240 void __sched_core_account_forceidle(struct rq *rq)  in __sched_core_account_forceidle()  argument
242 	const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));  in __sched_core_account_forceidle()
243 	u64 delta, now = rq_clock(rq->core);  in __sched_core_account_forceidle()
244 	struct rq *rq_i;  in __sched_core_account_forceidle()
248 	lockdep_assert_rq_held(rq);  in __sched_core_account_forceidle()
250 	WARN_ON_ONCE(!rq->core->core_forceidle_count);  in __sched_core_account_forceidle()
252 	if (rq->core->core_forceidle_start == 0)  in __sched_core_account_forceidle()
255 	delta = now - rq->core->core_forceidle_start;  in __sched_core_account_forceidle()
259 	rq->core->core_forceidle_start = now;  in __sched_core_account_forceidle()
261 	if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) {  in __sched_core_account_forceidle()
263 	} else if (rq->core->core_forceidle_count > 1 ||  in __sched_core_account_forceidle()
264 		   rq->core->core_forceidle_occupation > 1) {  in __sched_core_account_forceidle()
270 		delta *= rq->core->core_forceidle_count;  in __sched_core_account_forceidle()
271 		delta = div_u64(delta, rq->core->core_forceidle_occupation);  in __sched_core_account_forceidle()
289 void __sched_core_tick(struct rq *rq)  in __sched_core_tick()  argument
291 	if (!rq->core->core_forceidle_count)  in __sched_core_tick()
294 	if (rq != rq->core)  in __sched_core_tick()
295 		update_rq_clock(rq->core);  in __sched_core_tick()
297 	__sched_core_account_forceidle(rq);  in __sched_core_tick()