Lines Matching refs:now
564 u64 now; member
955 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now) in ioc_refresh_vrate() argument
957 s64 pleft = ioc->period_at + ioc->period_us - now->now; in ioc_refresh_vrate()
986 static void ioc_now(struct ioc *ioc, struct ioc_now *now) in ioc_now() argument
990 now->now_ns = ktime_get(); in ioc_now()
991 now->now = ktime_to_us(now->now_ns); in ioc_now()
992 now->vrate = atomic64_read(&ioc->vtime_rate); in ioc_now()
1004 now->vnow = ioc->period_at_vtime + in ioc_now()
1005 (now->now - ioc->period_at) * now->vrate; in ioc_now()
1009 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now) in ioc_start_period() argument
1014 ioc->period_at = now->now; in ioc_start_period()
1015 ioc->period_at_vtime = now->vnow; in ioc_start_period()
1028 bool save, struct ioc_now *now) in __propagate_weights() argument
1049 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime); in __propagate_weights()
1103 bool save, struct ioc_now *now) in propagate_weights() argument
1105 __propagate_weights(iocg, active, inuse, save, now); in propagate_weights()
1189 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now) in weight_updated() argument
1200 propagate_weights(iocg, weight, iocg->inuse, true, now); in weight_updated()
1204 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now) in iocg_activate() argument
1216 ioc_now(ioc, now); in iocg_activate()
1229 ioc_now(ioc, now); in iocg_activate()
1250 vtarget = now->vnow - ioc->margins.target; in iocg_activate()
1266 iocg->last_inuse ?: iocg->weight, true, now); in iocg_activate()
1268 TRACE_IOCG_PATH(iocg_activate, iocg, now, in iocg_activate()
1271 iocg->activated_at = now->now; in iocg_activate()
1275 ioc->dfgv_period_at = now->now; in iocg_activate()
1277 ioc_start_period(ioc, now); in iocg_activate()
1289 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now) in iocg_kick_delay() argument
1300 tdelta = now->now - iocg->delay_at; in iocg_kick_delay()
1309 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow; in iocg_kick_delay()
1326 iocg->delay_at = now->now; in iocg_kick_delay()
1332 iocg->indelay_since = now->now; in iocg_kick_delay()
1337 iocg->local_stat.indelay_us += now->now - iocg->indelay_since; in iocg_kick_delay()
1347 struct ioc_now *now) in iocg_incur_debt() argument
1360 iocg->indebt_since = now->now; in iocg_incur_debt()
1361 propagate_weights(iocg, iocg->active, 0, false, now); in iocg_incur_debt()
1372 struct ioc_now *now) in iocg_pay_debt() argument
1385 iocg->local_stat.indebt_us += now->now - iocg->indebt_since; in iocg_pay_debt()
1389 false, now); in iocg_pay_debt()
1426 struct ioc_now *now) in iocg_kick_waitq() argument
1437 vbudget = now->vnow - atomic64_read(&iocg->vtime); in iocg_kick_waitq()
1449 iocg_pay_debt(iocg, abs_vpay, now); in iocg_kick_waitq()
1454 iocg_kick_delay(iocg, now); in iocg_kick_waitq()
1479 iocg->local_stat.wait_us += now->now - iocg->wait_since; in iocg_kick_waitq()
1486 iocg->wait_since = now->now; in iocg_kick_waitq()
1493 expires = now->now_ns + in iocg_kick_waitq()
1512 struct ioc_now now; in iocg_waitq_timer_fn() local
1515 ioc_now(iocg->ioc, &now); in iocg_waitq_timer_fn()
1518 iocg_kick_waitq(iocg, pay_debt, &now); in iocg_waitq_timer_fn()
1608 static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now) in iocg_flush_stat_one() argument
1658 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now) in iocg_flush_stat() argument
1665 iocg_flush_stat_one(iocg, now); in iocg_flush_stat()
1671 iocg_flush_stat_one(iocg, now); in iocg_flush_stat()
1682 u32 usage, struct ioc_now *now) in hweight_after_donation() argument
1694 time_after64(vtime, now->vnow - ioc->margins.min)) in hweight_after_donation()
1698 excess = now->vnow - vtime - ioc->margins.target; in hweight_after_donation()
1722 delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime), in hweight_after_donation()
1723 now->vnow - ioc->period_at_vtime); in hweight_after_donation()
1787 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now) in transfer_surpluses() argument
1989 TRACE_IOCG_PATH(inuse_transfer, iocg, now, in transfer_surpluses()
1994 __propagate_weights(iocg, iocg->active, inuse, true, now); in transfer_surpluses()
2014 struct ioc_now *now) in ioc_forgive_debts() argument
2021 ioc->dfgv_period_at = now->now; in ioc_forgive_debts()
2037 if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD)) in ioc_forgive_debts()
2044 dur = now->now - ioc->dfgv_period_at; in ioc_forgive_debts()
2047 ioc->dfgv_period_at = now->now; in ioc_forgive_debts()
2084 iocg_kick_waitq(iocg, true, now); in ioc_forgive_debts()
2086 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct, in ioc_forgive_debts()
2098 struct ioc_now now; in ioc_timer_fn() local
2114 ioc_now(ioc, &now); in ioc_timer_fn()
2116 period_vtime = now.vnow - ioc->period_at_vtime; in ioc_timer_fn()
2137 iocg->local_stat.wait_us += now.now - iocg->wait_since; in ioc_timer_fn()
2138 iocg->wait_since = now.now; in ioc_timer_fn()
2142 now.now - iocg->indebt_since; in ioc_timer_fn()
2143 iocg->indebt_since = now.now; in ioc_timer_fn()
2147 now.now - iocg->indelay_since; in ioc_timer_fn()
2148 iocg->indelay_since = now.now; in ioc_timer_fn()
2154 iocg_kick_waitq(iocg, true, &now); in ioc_timer_fn()
2168 excess = now.vnow - vtime - ioc->margins.target; in ioc_timer_fn()
2177 __propagate_weights(iocg, 0, 0, false, &now); in ioc_timer_fn()
2189 iocg_flush_stat(&ioc->active_iocgs, &now); in ioc_timer_fn()
2213 time_after64(vtime, now.vnow - in ioc_timer_fn()
2215 time_before64(vdone, now.vnow - period_vtime)) in ioc_timer_fn()
2234 usage_dur = max_t(u64, now.now - iocg->activated_at, 1); in ioc_timer_fn()
2236 usage_dur = max_t(u64, now.now - ioc->period_at, 1); in ioc_timer_fn()
2247 time_before64(vtime, now.vnow - ioc->margins.low))) { in ioc_timer_fn()
2257 usage, &now); in ioc_timer_fn()
2280 TRACE_IOCG_PATH(inuse_shortage, iocg, &now, in ioc_timer_fn()
2285 iocg->active, true, &now); in ioc_timer_fn()
2295 transfer_surpluses(&surpluses, &now); in ioc_timer_fn()
2397 ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now); in ioc_timer_fn()
2407 ioc_start_period(ioc, &now); in ioc_timer_fn()
2414 ioc_refresh_vrate(ioc, &now); in ioc_timer_fn()
2421 u64 abs_cost, struct ioc_now *now) in adjust_inuse_and_calc_cost() argument
2433 margin = now->vnow - vtime - cost; in adjust_inuse_and_calc_cost()
2466 propagate_weights(iocg, iocg->active, new_inuse, true, now); in adjust_inuse_and_calc_cost()
2469 } while (time_after64(vtime + cost, now->vnow) && in adjust_inuse_and_calc_cost()
2474 TRACE_IOCG_PATH(inuse_adjust, iocg, now, in adjust_inuse_and_calc_cost()
2559 struct ioc_now now; in ioc_rqos_throttle() local
2574 if (!iocg_activate(iocg, &now)) in ioc_rqos_throttle()
2579 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now); in ioc_rqos_throttle()
2587 time_before_eq64(vtime + cost, now.vnow)) { in ioc_rqos_throttle()
2635 iocg_incur_debt(iocg, abs_cost, &now); in ioc_rqos_throttle()
2636 if (iocg_kick_delay(iocg, &now)) in ioc_rqos_throttle()
2651 &now); in ioc_rqos_throttle()
2674 iocg_kick_waitq(iocg, ioc_locked, &now); in ioc_rqos_throttle()
2695 struct ioc_now now; in ioc_rqos_merge() local
2707 ioc_now(ioc, &now); in ioc_rqos_merge()
2710 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now); in ioc_rqos_merge()
2722 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) { in ioc_rqos_merge()
2736 iocg_incur_debt(iocg, abs_cost, &now); in ioc_rqos_merge()
2737 if (iocg_kick_delay(iocg, &now)) in ioc_rqos_merge()
2935 struct ioc_now now; in ioc_pd_init() local
2939 ioc_now(ioc, &now); in ioc_pd_init()
2942 atomic64_set(&iocg->vtime, now.vnow); in ioc_pd_init()
2943 atomic64_set(&iocg->done_vtime, now.vnow); in ioc_pd_init()
2963 weight_updated(iocg, &now); in ioc_pd_init()
2977 struct ioc_now now; in ioc_pd_free() local
2979 ioc_now(ioc, &now); in ioc_pd_free()
2980 propagate_weights(iocg, 0, 0, false, &now); in ioc_pd_free()
3054 struct ioc_now now; in ioc_weight_write() local
3075 ioc_now(iocg->ioc, &now); in ioc_weight_write()
3076 weight_updated(iocg, &now); in ioc_weight_write()
3102 ioc_now(iocg->ioc, &now); in ioc_weight_write()
3103 weight_updated(iocg, &now); in ioc_weight_write()