12 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
18 	struct rt_bandwidth *rt_b =  in sched_rt_period_timer()  local
26 		overrun = hrtimer_forward(timer, now, rt_b->rt_period);  in sched_rt_period_timer()
31 		idle = do_sched_rt_period_timer(rt_b, overrun);  in sched_rt_period_timer()
37 void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)  in init_rt_bandwidth()  argument
39 	rt_b->rt_period = ns_to_ktime(period);  in init_rt_bandwidth()
40 	rt_b->rt_runtime = runtime;  in init_rt_bandwidth()
42 	raw_spin_lock_init(&rt_b->rt_runtime_lock);  in init_rt_bandwidth()
44 	hrtimer_init(&rt_b->rt_period_timer,  in init_rt_bandwidth()
46 	rt_b->rt_period_timer.function = sched_rt_period_timer;  in init_rt_bandwidth()
49 static void start_rt_bandwidth(struct rt_bandwidth *rt_b)  in start_rt_bandwidth()  argument
51 	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)  in start_rt_bandwidth()
54 	if (hrtimer_active(&rt_b->rt_period_timer))  in start_rt_bandwidth()
57 	raw_spin_lock(&rt_b->rt_runtime_lock);  in start_rt_bandwidth()
58 	start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);  in start_rt_bandwidth()
59 	raw_spin_unlock(&rt_b->rt_runtime_lock);  in start_rt_bandwidth()
90 static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)  in destroy_rt_bandwidth()  argument
92 	hrtimer_cancel(&rt_b->rt_period_timer);  in destroy_rt_bandwidth()
485 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()  argument
487 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];  in sched_rt_period_rt_rq()
552 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()  argument
570 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in do_balance_runtime()  local
577 	raw_spin_lock(&rt_b->rt_runtime_lock);  in do_balance_runtime()
578 	rt_period = ktime_to_ns(rt_b->rt_period);  in do_balance_runtime()
580 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in do_balance_runtime()
615 	raw_spin_unlock(&rt_b->rt_runtime_lock);  in do_balance_runtime()
633 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __disable_runtime()  local
637 		raw_spin_lock(&rt_b->rt_runtime_lock);  in __disable_runtime()
645 				rt_rq->rt_runtime == rt_b->rt_runtime)  in __disable_runtime()
654 		want = rt_b->rt_runtime - rt_rq->rt_runtime;  in __disable_runtime()
660 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in __disable_runtime()
698 		raw_spin_unlock(&rt_b->rt_runtime_lock);  in __disable_runtime()
723 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __enable_runtime()  local
725 		raw_spin_lock(&rt_b->rt_runtime_lock);  in __enable_runtime()
727 		rt_rq->rt_runtime = rt_b->rt_runtime;  in __enable_runtime()
731 		raw_spin_unlock(&rt_b->rt_runtime_lock);  in __enable_runtime()
788 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)  in do_sched_rt_period_timer()  argument
804 	if (rt_b == &root_task_group.rt_bandwidth)  in do_sched_rt_period_timer()
809 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);  in do_sched_rt_period_timer()
848 	if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))  in do_sched_rt_period_timer()
882 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_runtime_exceeded()  local
888 		if (likely(rt_b->rt_runtime)) {  in sched_rt_runtime_exceeded()