Lines Matching refs:cfs_b
2063 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) in __refill_cfs_bandwidth_runtime() argument
2067 if (cfs_b->quota == RUNTIME_INF) in __refill_cfs_bandwidth_runtime()
2071 cfs_b->runtime = cfs_b->quota; in __refill_cfs_bandwidth_runtime()
2072 cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); in __refill_cfs_bandwidth_runtime()
2093 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); in assign_cfs_rq_runtime() local
2099 raw_spin_lock(&cfs_b->lock); in assign_cfs_rq_runtime()
2100 if (cfs_b->quota == RUNTIME_INF) in assign_cfs_rq_runtime()
2109 if (!cfs_b->timer_active) { in assign_cfs_rq_runtime()
2110 __refill_cfs_bandwidth_runtime(cfs_b); in assign_cfs_rq_runtime()
2111 __start_cfs_bandwidth(cfs_b); in assign_cfs_rq_runtime()
2114 if (cfs_b->runtime > 0) { in assign_cfs_rq_runtime()
2115 amount = min(cfs_b->runtime, min_amount); in assign_cfs_rq_runtime()
2116 cfs_b->runtime -= amount; in assign_cfs_rq_runtime()
2117 cfs_b->idle = 0; in assign_cfs_rq_runtime()
2120 expires = cfs_b->runtime_expires; in assign_cfs_rq_runtime()
2121 raw_spin_unlock(&cfs_b->lock); in assign_cfs_rq_runtime()
2141 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in expire_cfs_rq_runtime() local
2160 if ((s64)(cfs_rq->runtime_expires - cfs_b->runtime_expires) >= 0) { in expire_cfs_rq_runtime()
2258 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in throttle_cfs_rq() local
2289 raw_spin_lock(&cfs_b->lock); in throttle_cfs_rq()
2290 list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); in throttle_cfs_rq()
2291 raw_spin_unlock(&cfs_b->lock); in throttle_cfs_rq()
2297 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in unthrottle_cfs_rq() local
2305 raw_spin_lock(&cfs_b->lock); in unthrottle_cfs_rq()
2306 cfs_b->throttled_time += rq->clock - cfs_rq->throttled_clock; in unthrottle_cfs_rq()
2308 raw_spin_unlock(&cfs_b->lock); in unthrottle_cfs_rq()
2339 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, in distribute_cfs_runtime() argument
2346 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, in distribute_cfs_runtime()
2383 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) in do_sched_cfs_period_timer() argument
2388 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_period_timer()
2390 if (cfs_b->quota == RUNTIME_INF) in do_sched_cfs_period_timer()
2393 throttled = !list_empty(&cfs_b->throttled_cfs_rq); in do_sched_cfs_period_timer()
2395 idle = cfs_b->idle && !throttled; in do_sched_cfs_period_timer()
2396 cfs_b->nr_periods += overrun; in do_sched_cfs_period_timer()
2402 __refill_cfs_bandwidth_runtime(cfs_b); in do_sched_cfs_period_timer()
2406 cfs_b->idle = 1; in do_sched_cfs_period_timer()
2411 cfs_b->nr_throttled += overrun; in do_sched_cfs_period_timer()
2419 runtime = cfs_b->runtime; in do_sched_cfs_period_timer()
2420 runtime_expires = cfs_b->runtime_expires; in do_sched_cfs_period_timer()
2421 cfs_b->runtime = 0; in do_sched_cfs_period_timer()
2429 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_period_timer()
2431 runtime = distribute_cfs_runtime(cfs_b, runtime, in do_sched_cfs_period_timer()
2433 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_period_timer()
2435 throttled = !list_empty(&cfs_b->throttled_cfs_rq); in do_sched_cfs_period_timer()
2439 cfs_b->runtime = runtime; in do_sched_cfs_period_timer()
2446 cfs_b->idle = 0; in do_sched_cfs_period_timer()
2449 cfs_b->timer_active = 0; in do_sched_cfs_period_timer()
2450 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_period_timer()
2463 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) in runtime_refresh_within() argument
2465 struct hrtimer *refresh_timer = &cfs_b->period_timer; in runtime_refresh_within()
2480 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) in start_cfs_slack_bandwidth() argument
2485 if (runtime_refresh_within(cfs_b, min_left)) in start_cfs_slack_bandwidth()
2488 start_bandwidth_timer(&cfs_b->slack_timer, in start_cfs_slack_bandwidth()
2495 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in __return_cfs_rq_runtime() local
2501 raw_spin_lock(&cfs_b->lock); in __return_cfs_rq_runtime()
2502 if (cfs_b->quota != RUNTIME_INF && in __return_cfs_rq_runtime()
2503 cfs_rq->runtime_expires == cfs_b->runtime_expires) { in __return_cfs_rq_runtime()
2504 cfs_b->runtime += slack_runtime; in __return_cfs_rq_runtime()
2507 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && in __return_cfs_rq_runtime()
2508 !list_empty(&cfs_b->throttled_cfs_rq)) in __return_cfs_rq_runtime()
2509 start_cfs_slack_bandwidth(cfs_b); in __return_cfs_rq_runtime()
2511 raw_spin_unlock(&cfs_b->lock); in __return_cfs_rq_runtime()
2532 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) in do_sched_cfs_slack_timer() argument
2538 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) in do_sched_cfs_slack_timer()
2541 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_slack_timer()
2542 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) { in do_sched_cfs_slack_timer()
2543 runtime = cfs_b->runtime; in do_sched_cfs_slack_timer()
2544 cfs_b->runtime = 0; in do_sched_cfs_slack_timer()
2546 expires = cfs_b->runtime_expires; in do_sched_cfs_slack_timer()
2547 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_slack_timer()
2552 runtime = distribute_cfs_runtime(cfs_b, runtime, expires); in do_sched_cfs_slack_timer()
2554 raw_spin_lock(&cfs_b->lock); in do_sched_cfs_slack_timer()
2555 if (expires == cfs_b->runtime_expires) in do_sched_cfs_slack_timer()
2556 cfs_b->runtime = runtime; in do_sched_cfs_slack_timer()
2557 raw_spin_unlock(&cfs_b->lock); in do_sched_cfs_slack_timer()
2604 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
2605 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b);
2609 struct cfs_bandwidth *cfs_b = in sched_cfs_slack_timer() local
2611 do_sched_cfs_slack_timer(cfs_b); in sched_cfs_slack_timer()
2618 struct cfs_bandwidth *cfs_b = in sched_cfs_period_timer() local
2626 overrun = hrtimer_forward(timer, now, cfs_b->period); in sched_cfs_period_timer()
2631 idle = do_sched_cfs_period_timer(cfs_b, overrun); in sched_cfs_period_timer()
2637 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in init_cfs_bandwidth() argument
2639 raw_spin_lock_init(&cfs_b->lock); in init_cfs_bandwidth()
2640 cfs_b->runtime = 0; in init_cfs_bandwidth()
2641 cfs_b->quota = RUNTIME_INF; in init_cfs_bandwidth()
2642 cfs_b->period = ns_to_ktime(default_cfs_period()); in init_cfs_bandwidth()
2644 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); in init_cfs_bandwidth()
2645 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in init_cfs_bandwidth()
2646 cfs_b->period_timer.function = sched_cfs_period_timer; in init_cfs_bandwidth()
2647 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in init_cfs_bandwidth()
2648 cfs_b->slack_timer.function = sched_cfs_slack_timer; in init_cfs_bandwidth()
2658 void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in __start_cfs_bandwidth() argument
2666 while (unlikely(hrtimer_active(&cfs_b->period_timer))) { in __start_cfs_bandwidth()
2667 raw_spin_unlock(&cfs_b->lock); in __start_cfs_bandwidth()
2669 hrtimer_cancel(&cfs_b->period_timer); in __start_cfs_bandwidth()
2671 raw_spin_lock(&cfs_b->lock); in __start_cfs_bandwidth()
2673 if (cfs_b->timer_active) in __start_cfs_bandwidth()
2677 cfs_b->timer_active = 1; in __start_cfs_bandwidth()
2678 start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period); in __start_cfs_bandwidth()
2681 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) in destroy_cfs_bandwidth() argument
2683 hrtimer_cancel(&cfs_b->period_timer); in destroy_cfs_bandwidth()
2684 hrtimer_cancel(&cfs_b->slack_timer); in destroy_cfs_bandwidth()
2692 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in unthrottle_offline_cfs_rqs() local
2701 cfs_rq->runtime_remaining = cfs_b->quota; in unthrottle_offline_cfs_rqs()
2735 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} in init_cfs_bandwidth() argument
2745 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} in destroy_cfs_bandwidth() argument