Lines Matching refs:iolat
192 static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat) in lat_to_blkg() argument
194 return pd_to_blkg(&iolat->pd); in lat_to_blkg()
197 static inline void latency_stat_init(struct iolatency_grp *iolat, in latency_stat_init() argument
200 if (iolat->ssd) { in latency_stat_init()
207 static inline void latency_stat_sum(struct iolatency_grp *iolat, in latency_stat_sum() argument
211 if (iolat->ssd) { in latency_stat_sum()
218 static inline void latency_stat_record_time(struct iolatency_grp *iolat, in latency_stat_record_time() argument
221 struct latency_stat *stat = get_cpu_ptr(iolat->stats); in latency_stat_record_time()
222 if (iolat->ssd) { in latency_stat_record_time()
223 if (req_time >= iolat->min_lat_nsec) in latency_stat_record_time()
231 static inline bool latency_sum_ok(struct iolatency_grp *iolat, in latency_sum_ok() argument
234 if (iolat->ssd) { in latency_sum_ok()
239 return stat->rqs.mean <= iolat->min_lat_nsec; in latency_sum_ok()
242 static inline u64 latency_stat_samples(struct iolatency_grp *iolat, in latency_stat_samples() argument
245 if (iolat->ssd) in latency_stat_samples()
250 static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat, in iolat_update_total_lat_avg() argument
255 if (iolat->ssd) in iolat_update_total_lat_avg()
266 div64_u64(iolat->cur_win_nsec, in iolat_update_total_lat_avg()
268 iolat->lat_avg = calc_load(iolat->lat_avg, in iolat_update_total_lat_avg()
281 struct iolatency_grp *iolat = private_data; in iolat_acquire_inflight() local
282 return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth); in iolat_acquire_inflight()
286 struct iolatency_grp *iolat, in __blkcg_iolatency_throttle() argument
290 struct rq_wait *rqw = &iolat->rq_wait; in __blkcg_iolatency_throttle()
291 unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay); in __blkcg_iolatency_throttle()
308 rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb); in __blkcg_iolatency_throttle()
370 static void scale_change(struct iolatency_grp *iolat, bool up) in scale_change() argument
372 unsigned long qd = iolat->blkiolat->rqos.q->nr_requests; in scale_change()
374 unsigned long old = iolat->rq_depth.max_depth; in scale_change()
380 if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat))) in scale_change()
386 iolat->rq_depth.max_depth = old; in scale_change()
387 wake_up_all(&iolat->rq_wait.wait); in scale_change()
391 iolat->rq_depth.max_depth = max(old, 1UL); in scale_change()
396 static void check_scale_change(struct iolatency_grp *iolat) in check_scale_change() argument
401 unsigned int our_cookie = atomic_read(&iolat->scale_cookie); in check_scale_change()
406 if (lat_to_blkg(iolat)->parent == NULL) in check_scale_change()
409 parent = blkg_to_lat(lat_to_blkg(iolat)->parent); in check_scale_change()
424 old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie); in check_scale_change()
430 if (direction < 0 && iolat->min_lat_nsec) { in check_scale_change()
433 if (!scale_lat || iolat->min_lat_nsec <= scale_lat) in check_scale_change()
444 if (iolat->nr_samples <= samples_thresh) in check_scale_change()
449 if (iolat->rq_depth.max_depth == 1 && direction < 0) { in check_scale_change()
450 blkcg_use_delay(lat_to_blkg(iolat)); in check_scale_change()
456 blkcg_clear_delay(lat_to_blkg(iolat)); in check_scale_change()
457 iolat->rq_depth.max_depth = UINT_MAX; in check_scale_change()
458 wake_up_all(&iolat->rq_wait.wait); in check_scale_change()
462 scale_change(iolat, direction > 0); in check_scale_change()
475 struct iolatency_grp *iolat = blkg_to_lat(blkg); in blkcg_iolatency_throttle() local
476 if (!iolat) { in blkcg_iolatency_throttle()
481 check_scale_change(iolat); in blkcg_iolatency_throttle()
482 __blkcg_iolatency_throttle(rqos, iolat, issue_as_root, in blkcg_iolatency_throttle()
490 static void iolatency_record_time(struct iolatency_grp *iolat, in iolatency_record_time() argument
512 if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) { in iolatency_record_time()
513 u64 sub = iolat->min_lat_nsec; in iolatency_record_time()
515 blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time); in iolatency_record_time()
519 latency_stat_record_time(iolat, req_time); in iolatency_record_time()
525 static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) in iolatency_check_latencies() argument
527 struct blkcg_gq *blkg = lat_to_blkg(iolat); in iolatency_check_latencies()
534 latency_stat_init(iolat, &stat); in iolatency_check_latencies()
538 s = per_cpu_ptr(iolat->stats, cpu); in iolatency_check_latencies()
539 latency_stat_sum(iolat, &stat, s); in iolatency_check_latencies()
540 latency_stat_init(iolat, s); in iolatency_check_latencies()
550 iolat_update_total_lat_avg(iolat, &stat); in iolatency_check_latencies()
553 if (latency_sum_ok(iolat, &stat) && in iolatency_check_latencies()
560 latency_stat_sum(iolat, &iolat->cur_stat, &stat); in iolatency_check_latencies()
561 lat_info->nr_samples -= iolat->nr_samples; in iolatency_check_latencies()
562 lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat); in iolatency_check_latencies()
563 iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat); in iolatency_check_latencies()
569 if (latency_sum_ok(iolat, &iolat->cur_stat) && in iolatency_check_latencies()
570 latency_sum_ok(iolat, &stat)) { in iolatency_check_latencies()
571 if (latency_stat_samples(iolat, &iolat->cur_stat) < in iolatency_check_latencies()
574 if (lat_info->scale_grp == iolat) { in iolatency_check_latencies()
576 scale_cookie_change(iolat->blkiolat, lat_info, true); in iolatency_check_latencies()
579 lat_info->scale_lat >= iolat->min_lat_nsec) { in iolatency_check_latencies()
582 lat_info->scale_lat > iolat->min_lat_nsec) { in iolatency_check_latencies()
583 WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec); in iolatency_check_latencies()
584 lat_info->scale_grp = iolat; in iolatency_check_latencies()
586 scale_cookie_change(iolat->blkiolat, lat_info, false); in iolatency_check_latencies()
588 latency_stat_init(iolat, &iolat->cur_stat); in iolatency_check_latencies()
597 struct iolatency_grp *iolat; in blkcg_iolatency_done_bio() local
607 iolat = blkg_to_lat(bio->bi_blkg); in blkcg_iolatency_done_bio()
608 if (!iolat) in blkcg_iolatency_done_bio()
611 if (!iolat->blkiolat->enabled) in blkcg_iolatency_done_bio()
616 iolat = blkg_to_lat(blkg); in blkcg_iolatency_done_bio()
617 if (!iolat) { in blkcg_iolatency_done_bio()
621 rqw = &iolat->rq_wait; in blkcg_iolatency_done_bio()
629 if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) { in blkcg_iolatency_done_bio()
630 iolatency_record_time(iolat, &bio->bi_issue, now, in blkcg_iolatency_done_bio()
632 window_start = atomic64_read(&iolat->window_start); in blkcg_iolatency_done_bio()
634 (now - window_start) >= iolat->cur_win_nsec) { in blkcg_iolatency_done_bio()
635 if (atomic64_cmpxchg(&iolat->window_start, in blkcg_iolatency_done_bio()
637 iolatency_check_latencies(iolat, now); in blkcg_iolatency_done_bio()
671 struct iolatency_grp *iolat; in blkiolatency_timer_fn() local
683 iolat = blkg_to_lat(blkg); in blkiolatency_timer_fn()
684 if (!iolat) in blkiolatency_timer_fn()
687 lat_info = &iolat->child_lat; in blkiolatency_timer_fn()
702 scale_cookie_change(iolat->blkiolat, lat_info, true); in blkiolatency_timer_fn()
796 struct iolatency_grp *iolat = blkg_to_lat(blkg); in iolatency_set_min_lat_nsec() local
797 struct blk_iolatency *blkiolat = iolat->blkiolat; in iolatency_set_min_lat_nsec()
798 u64 oldval = iolat->min_lat_nsec; in iolatency_set_min_lat_nsec()
800 iolat->min_lat_nsec = val; in iolatency_set_min_lat_nsec()
801 iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE); in iolatency_set_min_lat_nsec()
802 iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec, in iolatency_set_min_lat_nsec()
819 struct iolatency_grp *iolat = blkg_to_lat(blkg->parent); in iolatency_clear_scaling() local
821 if (!iolat) in iolatency_clear_scaling()
824 lat_info = &iolat->child_lat; in iolatency_clear_scaling()
840 struct iolatency_grp *iolat; in iolatency_set_limit() local
850 iolat = blkg_to_lat(ctx.blkg); in iolatency_set_limit()
877 oldval = iolat->min_lat_nsec; in iolatency_set_limit()
880 if (oldval != iolat->min_lat_nsec) in iolatency_set_limit()
891 struct iolatency_grp *iolat = pd_to_lat(pd); in iolatency_prfill_limit() local
894 if (!dname || !iolat->min_lat_nsec) in iolatency_prfill_limit()
897 dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC)); in iolatency_prfill_limit()
909 static bool iolatency_ssd_stat(struct iolatency_grp *iolat, struct seq_file *s) in iolatency_ssd_stat() argument
914 latency_stat_init(iolat, &stat); in iolatency_ssd_stat()
918 s = per_cpu_ptr(iolat->stats, cpu); in iolatency_ssd_stat()
919 latency_stat_sum(iolat, &stat, s); in iolatency_ssd_stat()
923 if (iolat->rq_depth.max_depth == UINT_MAX) in iolatency_ssd_stat()
931 iolat->rq_depth.max_depth); in iolatency_ssd_stat()
937 struct iolatency_grp *iolat = pd_to_lat(pd); in iolatency_pd_stat() local
944 if (iolat->ssd) in iolatency_pd_stat()
945 return iolatency_ssd_stat(iolat, s); in iolatency_pd_stat()
947 avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC); in iolatency_pd_stat()
948 cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC); in iolatency_pd_stat()
949 if (iolat->rq_depth.max_depth == UINT_MAX) in iolatency_pd_stat()
954 iolat->rq_depth.max_depth, avg_lat, cur_win); in iolatency_pd_stat()
962 struct iolatency_grp *iolat; in iolatency_pd_alloc() local
964 iolat = kzalloc_node(sizeof(*iolat), gfp, q->node); in iolatency_pd_alloc()
965 if (!iolat) in iolatency_pd_alloc()
967 iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat), in iolatency_pd_alloc()
969 if (!iolat->stats) { in iolatency_pd_alloc()
970 kfree(iolat); in iolatency_pd_alloc()
973 return &iolat->pd; in iolatency_pd_alloc()
978 struct iolatency_grp *iolat = pd_to_lat(pd); in iolatency_pd_init() local
979 struct blkcg_gq *blkg = lat_to_blkg(iolat); in iolatency_pd_init()
986 iolat->ssd = true; in iolatency_pd_init()
988 iolat->ssd = false; in iolatency_pd_init()
992 stat = per_cpu_ptr(iolat->stats, cpu); in iolatency_pd_init()
993 latency_stat_init(iolat, stat); in iolatency_pd_init()
996 latency_stat_init(iolat, &iolat->cur_stat); in iolatency_pd_init()
997 rq_wait_init(&iolat->rq_wait); in iolatency_pd_init()
998 spin_lock_init(&iolat->child_lat.lock); in iolatency_pd_init()
999 iolat->rq_depth.queue_depth = blkg->q->nr_requests; in iolatency_pd_init()
1000 iolat->rq_depth.max_depth = UINT_MAX; in iolatency_pd_init()
1001 iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth; in iolatency_pd_init()
1002 iolat->blkiolat = blkiolat; in iolatency_pd_init()
1003 iolat->cur_win_nsec = 100 * NSEC_PER_MSEC; in iolatency_pd_init()
1004 atomic64_set(&iolat->window_start, now); in iolatency_pd_init()
1012 atomic_set(&iolat->scale_cookie, in iolatency_pd_init()
1015 atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE); in iolatency_pd_init()
1018 atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE); in iolatency_pd_init()
1023 struct iolatency_grp *iolat = pd_to_lat(pd); in iolatency_pd_offline() local
1024 struct blkcg_gq *blkg = lat_to_blkg(iolat); in iolatency_pd_offline()
1032 struct iolatency_grp *iolat = pd_to_lat(pd); in iolatency_pd_free() local
1033 free_percpu(iolat->stats); in iolatency_pd_free()
1034 kfree(iolat); in iolatency_pd_free()