Lines Matching refs:rwb
78 static inline bool rwb_enabled(struct rq_wb *rwb) in rwb_enabled() argument
80 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && in rwb_enabled()
81 rwb->wb_normal != 0; in rwb_enabled()
84 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) in wb_timestamp() argument
86 if (rwb_enabled(rwb)) { in wb_timestamp()
98 static bool wb_recent_wait(struct rq_wb *rwb) in wb_recent_wait() argument
100 struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb; in wb_recent_wait()
105 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, in get_rq_wait() argument
109 return &rwb->rq_wait[WBT_RWQ_KSWAPD]; in get_rq_wait()
111 return &rwb->rq_wait[WBT_RWQ_DISCARD]; in get_rq_wait()
113 return &rwb->rq_wait[WBT_RWQ_BG]; in get_rq_wait()
116 static void rwb_wake_all(struct rq_wb *rwb) in rwb_wake_all() argument
121 struct rq_wait *rqw = &rwb->rq_wait[i]; in rwb_wake_all()
128 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, in wbt_rqw_done() argument
139 if (unlikely(!rwb_enabled(rwb))) { in wbt_rqw_done()
140 rwb_wake_all(rwb); in wbt_rqw_done()
150 limit = rwb->wb_background; in wbt_rqw_done()
151 else if (rwb->wc && !wb_recent_wait(rwb)) in wbt_rqw_done()
154 limit = rwb->wb_normal; in wbt_rqw_done()
165 if (!inflight || diff >= rwb->wb_background / 2) in wbt_rqw_done()
172 struct rq_wb *rwb = RQWB(rqos); in __wbt_done() local
178 rqw = get_rq_wait(rwb, wb_acct); in __wbt_done()
179 wbt_rqw_done(rwb, rqw, wb_acct); in __wbt_done()
188 struct rq_wb *rwb = RQWB(rqos); in wbt_done() local
191 if (rwb->sync_cookie == rq) { in wbt_done()
192 rwb->sync_issue = 0; in wbt_done()
193 rwb->sync_cookie = NULL; in wbt_done()
197 wb_timestamp(rwb, &rwb->last_comp); in wbt_done()
199 WARN_ON_ONCE(rq == rwb->sync_cookie); in wbt_done()
217 static u64 rwb_sync_issue_lat(struct rq_wb *rwb) in rwb_sync_issue_lat() argument
219 u64 now, issue = READ_ONCE(rwb->sync_issue); in rwb_sync_issue_lat()
221 if (!issue || !rwb->sync_cookie) in rwb_sync_issue_lat()
235 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) in latency_exceeded() argument
237 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info; in latency_exceeded()
238 struct rq_depth *rqd = &rwb->rq_depth; in latency_exceeded()
250 thislat = rwb_sync_issue_lat(rwb); in latency_exceeded()
251 if (thislat > rwb->cur_win_nsec || in latency_exceeded()
252 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) { in latency_exceeded()
267 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) || in latency_exceeded()
268 wbt_inflight(rwb)) in latency_exceeded()
276 if (stat[READ].min > rwb->min_lat_nsec) { in latency_exceeded()
288 static void rwb_trace_step(struct rq_wb *rwb, const char *msg) in rwb_trace_step() argument
290 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info; in rwb_trace_step()
291 struct rq_depth *rqd = &rwb->rq_depth; in rwb_trace_step()
293 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec, in rwb_trace_step()
294 rwb->wb_background, rwb->wb_normal, rqd->max_depth); in rwb_trace_step()
297 static void calc_wb_limits(struct rq_wb *rwb) in calc_wb_limits() argument
299 if (rwb->min_lat_nsec == 0) { in calc_wb_limits()
300 rwb->wb_normal = rwb->wb_background = 0; in calc_wb_limits()
301 } else if (rwb->rq_depth.max_depth <= 2) { in calc_wb_limits()
302 rwb->wb_normal = rwb->rq_depth.max_depth; in calc_wb_limits()
303 rwb->wb_background = 1; in calc_wb_limits()
305 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2; in calc_wb_limits()
306 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4; in calc_wb_limits()
310 static void scale_up(struct rq_wb *rwb) in scale_up() argument
312 if (!rq_depth_scale_up(&rwb->rq_depth)) in scale_up()
314 calc_wb_limits(rwb); in scale_up()
315 rwb->unknown_cnt = 0; in scale_up()
316 rwb_wake_all(rwb); in scale_up()
317 rwb_trace_step(rwb, tracepoint_string("scale up")); in scale_up()
320 static void scale_down(struct rq_wb *rwb, bool hard_throttle) in scale_down() argument
322 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle)) in scale_down()
324 calc_wb_limits(rwb); in scale_down()
325 rwb->unknown_cnt = 0; in scale_down()
326 rwb_trace_step(rwb, tracepoint_string("scale down")); in scale_down()
329 static void rwb_arm_timer(struct rq_wb *rwb) in rwb_arm_timer() argument
331 struct rq_depth *rqd = &rwb->rq_depth; in rwb_arm_timer()
340 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4, in rwb_arm_timer()
347 rwb->cur_win_nsec = rwb->win_nsec; in rwb_arm_timer()
350 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec); in rwb_arm_timer()
355 struct rq_wb *rwb = cb->data; in wb_timer_fn() local
356 struct rq_depth *rqd = &rwb->rq_depth; in wb_timer_fn()
357 unsigned int inflight = wbt_inflight(rwb); in wb_timer_fn()
360 status = latency_exceeded(rwb, cb->stat); in wb_timer_fn()
362 trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step, in wb_timer_fn()
372 scale_down(rwb, true); in wb_timer_fn()
375 scale_up(rwb); in wb_timer_fn()
383 scale_up(rwb); in wb_timer_fn()
386 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP) in wb_timer_fn()
394 scale_up(rwb); in wb_timer_fn()
396 scale_down(rwb, false); in wb_timer_fn()
406 rwb_arm_timer(rwb); in wb_timer_fn()
409 static void wbt_update_limits(struct rq_wb *rwb) in wbt_update_limits() argument
411 struct rq_depth *rqd = &rwb->rq_depth; in wbt_update_limits()
417 calc_wb_limits(rwb); in wbt_update_limits()
419 rwb_wake_all(rwb); in wbt_update_limits()
441 static bool close_io(struct rq_wb *rwb) in close_io() argument
445 return time_before(now, rwb->last_issue + HZ / 10) || in close_io()
446 time_before(now, rwb->last_comp + HZ / 10); in close_io()
451 static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) in get_limit() argument
459 if (!rwb_enabled(rwb)) in get_limit()
463 return rwb->wb_background; in get_limit()
473 if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) in get_limit()
474 limit = rwb->rq_depth.max_depth; in get_limit()
475 else if ((rw & REQ_BACKGROUND) || close_io(rwb)) { in get_limit()
480 limit = rwb->wb_background; in get_limit()
482 limit = rwb->wb_normal; in get_limit()
488 struct rq_wb *rwb; member
496 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw)); in wbt_inflight_cb()
502 wbt_rqw_done(data->rwb, rqw, data->wb_acct); in wbt_cleanup_cb()
509 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, in __wbt_wait() argument
512 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); in __wbt_wait()
514 .rwb = rwb, in __wbt_wait()
522 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio) in wbt_should_throttle() argument
540 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) in bio_to_wbt_flags() argument
544 if (!rwb_enabled(rwb)) in bio_to_wbt_flags()
549 } else if (wbt_should_throttle(rwb, bio)) { in bio_to_wbt_flags()
561 struct rq_wb *rwb = RQWB(rqos); in wbt_cleanup() local
562 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio); in wbt_cleanup()
574 struct rq_wb *rwb = RQWB(rqos); in wbt_wait() local
577 flags = bio_to_wbt_flags(rwb, bio); in wbt_wait()
580 wb_timestamp(rwb, &rwb->last_issue); in wbt_wait()
584 __wbt_wait(rwb, flags, bio->bi_opf); in wbt_wait()
586 if (!blk_stat_is_active(rwb->cb)) in wbt_wait()
587 rwb_arm_timer(rwb); in wbt_wait()
592 struct rq_wb *rwb = RQWB(rqos); in wbt_track() local
593 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); in wbt_track()
598 struct rq_wb *rwb = RQWB(rqos); in wbt_issue() local
600 if (!rwb_enabled(rwb)) in wbt_issue()
610 if (wbt_is_read(rq) && !rwb->sync_issue) { in wbt_issue()
611 rwb->sync_cookie = rq; in wbt_issue()
612 rwb->sync_issue = rq->io_start_time_ns; in wbt_issue()
618 struct rq_wb *rwb = RQWB(rqos); in wbt_requeue() local
619 if (!rwb_enabled(rwb)) in wbt_requeue()
621 if (rq == rwb->sync_cookie) { in wbt_requeue()
622 rwb->sync_issue = 0; in wbt_requeue()
623 rwb->sync_cookie = NULL; in wbt_requeue()
690 struct rq_wb *rwb = RQWB(rqos); in wbt_exit() local
693 blk_stat_remove_callback(q, rwb->cb); in wbt_exit()
694 blk_stat_free_callback(rwb->cb); in wbt_exit()
695 kfree(rwb); in wbt_exit()
704 struct rq_wb *rwb; in wbt_disable_default() local
707 rwb = RQWB(rqos); in wbt_disable_default()
708 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { in wbt_disable_default()
709 blk_stat_deactivate(rwb->cb); in wbt_disable_default()
710 rwb->enable_state = WBT_STATE_OFF_DEFAULT; in wbt_disable_default()
719 struct rq_wb *rwb = RQWB(rqos); in wbt_curr_win_nsec_show() local
721 seq_printf(m, "%llu\n", rwb->cur_win_nsec); in wbt_curr_win_nsec_show()
728 struct rq_wb *rwb = RQWB(rqos); in wbt_enabled_show() local
730 seq_printf(m, "%d\n", rwb->enable_state); in wbt_enabled_show()
745 struct rq_wb *rwb = RQWB(rqos); in wbt_inflight_show() local
750 atomic_read(&rwb->rq_wait[i].inflight)); in wbt_inflight_show()
757 struct rq_wb *rwb = RQWB(rqos); in wbt_min_lat_nsec_show() local
759 seq_printf(m, "%lu\n", rwb->min_lat_nsec); in wbt_min_lat_nsec_show()
766 struct rq_wb *rwb = RQWB(rqos); in wbt_unknown_cnt_show() local
768 seq_printf(m, "%u\n", rwb->unknown_cnt); in wbt_unknown_cnt_show()
775 struct rq_wb *rwb = RQWB(rqos); in wbt_normal_show() local
777 seq_printf(m, "%u\n", rwb->wb_normal); in wbt_normal_show()
784 struct rq_wb *rwb = RQWB(rqos); in wbt_background_show() local
786 seq_printf(m, "%u\n", rwb->wb_background); in wbt_background_show()
819 struct rq_wb *rwb; in wbt_init() local
822 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); in wbt_init()
823 if (!rwb) in wbt_init()
826 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); in wbt_init()
827 if (!rwb->cb) { in wbt_init()
828 kfree(rwb); in wbt_init()
833 rq_wait_init(&rwb->rq_wait[i]); in wbt_init()
835 rwb->rqos.id = RQ_QOS_WBT; in wbt_init()
836 rwb->rqos.ops = &wbt_rqos_ops; in wbt_init()
837 rwb->rqos.q = q; in wbt_init()
838 rwb->last_comp = rwb->last_issue = jiffies; in wbt_init()
839 rwb->win_nsec = RWB_WINDOW_NSEC; in wbt_init()
840 rwb->enable_state = WBT_STATE_ON_DEFAULT; in wbt_init()
841 rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags); in wbt_init()
842 rwb->rq_depth.default_depth = RWB_DEF_DEPTH; in wbt_init()
843 rwb->min_lat_nsec = wbt_default_latency_nsec(q); in wbt_init()
845 wbt_queue_depth_changed(&rwb->rqos); in wbt_init()
850 rq_qos_add(q, &rwb->rqos); in wbt_init()
851 blk_stat_add_callback(q, rwb->cb); in wbt_init()