Lines Matching refs:rw
300 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw) in tg_bps_limit() argument
310 ret = tg->bps[rw][td->limit_index]; in tg_bps_limit()
314 tg->iops[rw][td->limit_index]) in tg_bps_limit()
320 if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] && in tg_bps_limit()
321 tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) { in tg_bps_limit()
324 adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td); in tg_bps_limit()
325 ret = min(tg->bps[rw][LIMIT_MAX], adjusted); in tg_bps_limit()
330 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw) in tg_iops_limit() argument
340 ret = tg->iops[rw][td->limit_index]; in tg_iops_limit()
344 tg->bps[rw][td->limit_index]) in tg_iops_limit()
350 if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] && in tg_iops_limit()
351 tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) { in tg_iops_limit()
354 adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td); in tg_iops_limit()
357 ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted); in tg_iops_limit()
496 int rw; in throtl_pd_alloc() local
510 for (rw = READ; rw <= WRITE; rw++) { in throtl_pd_alloc()
511 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
512 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
575 int rw; in tg_update_has_rules() local
577 for (rw = READ; rw <= WRITE; rw++) in tg_update_has_rules()
578 tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) || in tg_update_has_rules()
580 (tg_bps_limit(tg, rw) != U64_MAX || in tg_update_has_rules()
581 tg_iops_limit(tg, rw) != UINT_MAX)); in tg_update_has_rules()
773 bool rw, unsigned long start) in throtl_start_new_slice_with_credit() argument
775 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
776 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
778 atomic_set(&tg->io_split_cnt[rw], 0); in throtl_start_new_slice_with_credit()
786 if (time_after_eq(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
787 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
789 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
792 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
793 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
796 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw) in throtl_start_new_slice() argument
798 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
799 tg->io_disp[rw] = 0; in throtl_start_new_slice()
800 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
801 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
803 atomic_set(&tg->io_split_cnt[rw], 0); in throtl_start_new_slice()
807 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
808 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
811 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
814 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
817 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
820 throtl_set_slice_end(tg, rw, jiffy_end); in throtl_extend_slice()
823 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
824 tg->slice_end[rw], jiffies); in throtl_extend_slice()
828 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
830 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
837 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
842 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
849 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
860 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
862 time_elapsed = jiffies - tg->slice_start[rw]; in throtl_trim_slice()
868 tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices; in throtl_trim_slice()
872 io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices) / in throtl_trim_slice()
878 if (tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
879 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
881 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
883 if (tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
884 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
886 tg->io_disp[rw] = 0; in throtl_trim_slice()
888 tg->slice_start[rw] += nr_slices * tg->td->throtl_slice; in throtl_trim_slice()
892 rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim, in throtl_trim_slice()
893 tg->slice_start[rw], tg->slice_end[rw], jiffies); in throtl_trim_slice()
899 bool rw = bio_data_dir(bio); in tg_with_in_iops_limit() local
910 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_with_in_iops_limit()
930 if (tg->io_disp[rw] + 1 <= io_allowed) { in tg_with_in_iops_limit()
947 bool rw = bio_data_dir(bio); in tg_with_in_bps_limit() local
958 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_with_in_bps_limit()
968 if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) { in tg_with_in_bps_limit()
975 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_with_in_bps_limit()
998 bool rw = bio_data_dir(bio); in tg_may_dispatch() local
1000 u64 bps_limit = tg_bps_limit(tg, rw); in tg_may_dispatch()
1001 u32 iops_limit = tg_iops_limit(tg, rw); in tg_may_dispatch()
1009 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
1010 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
1026 if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) in tg_may_dispatch()
1027 throtl_start_new_slice(tg, rw); in tg_may_dispatch()
1029 if (time_before(tg->slice_end[rw], in tg_may_dispatch()
1031 throtl_extend_slice(tg, rw, in tg_may_dispatch()
1036 tg->io_disp[rw] += atomic_xchg(&tg->io_split_cnt[rw], 0); in tg_may_dispatch()
1050 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
1051 throtl_extend_slice(tg, rw, jiffies + max_wait); in tg_may_dispatch()
1058 bool rw = bio_data_dir(bio); in throtl_charge_bio() local
1062 tg->bytes_disp[rw] += bio_size; in throtl_charge_bio()
1063 tg->io_disp[rw]++; in throtl_charge_bio()
1064 tg->last_bytes_disp[rw] += bio_size; in throtl_charge_bio()
1065 tg->last_io_disp[rw]++; in throtl_charge_bio()
1090 bool rw = bio_data_dir(bio); in throtl_add_bio_tg() local
1093 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
1101 if (!sq->nr_queued[rw]) in throtl_add_bio_tg()
1104 throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); in throtl_add_bio_tg()
1106 sq->nr_queued[rw]++; in throtl_add_bio_tg()
1137 struct throtl_grp *parent_tg, bool rw) in start_parent_slice_with_credit() argument
1139 if (throtl_slice_used(parent_tg, rw)) { in start_parent_slice_with_credit()
1140 throtl_start_new_slice_with_credit(parent_tg, rw, in start_parent_slice_with_credit()
1141 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
1146 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
1160 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); in tg_dispatch_one_bio()
1161 sq->nr_queued[rw]--; in tg_dispatch_one_bio()
1173 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1174 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
1176 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1177 &parent_sq->queued[rw]); in tg_dispatch_one_bio()
1178 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1179 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1182 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1348 int rw; in blk_throtl_dispatch_work_fn() local
1353 for (rw = READ; rw <= WRITE; rw++) in blk_throtl_dispatch_work_fn()
1354 while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) in blk_throtl_dispatch_work_fn()
2140 int i, cpu, rw; in throtl_update_latency_buckets() local
2151 for (rw = READ; rw <= WRITE; rw++) { in throtl_update_latency_buckets()
2153 struct latency_bucket *tmp = &td->tmp_buckets[rw][i]; in throtl_update_latency_buckets()
2159 bucket = per_cpu_ptr(td->latency_buckets[rw], in throtl_update_latency_buckets()
2170 latency[rw] = tmp->total_latency; in throtl_update_latency_buckets()
2174 latency[rw] /= samples; in throtl_update_latency_buckets()
2175 if (latency[rw] == 0) in throtl_update_latency_buckets()
2177 avg_latency[rw][i].latency = latency[rw]; in throtl_update_latency_buckets()
2182 for (rw = READ; rw <= WRITE; rw++) { in throtl_update_latency_buckets()
2184 if (!avg_latency[rw][i].latency) { in throtl_update_latency_buckets()
2185 if (td->avg_buckets[rw][i].latency < last_latency[rw]) in throtl_update_latency_buckets()
2186 td->avg_buckets[rw][i].latency = in throtl_update_latency_buckets()
2187 last_latency[rw]; in throtl_update_latency_buckets()
2191 if (!td->avg_buckets[rw][i].valid) in throtl_update_latency_buckets()
2192 latency[rw] = avg_latency[rw][i].latency; in throtl_update_latency_buckets()
2194 latency[rw] = (td->avg_buckets[rw][i].latency * 7 + in throtl_update_latency_buckets()
2195 avg_latency[rw][i].latency) >> 3; in throtl_update_latency_buckets()
2197 td->avg_buckets[rw][i].latency = max(latency[rw], in throtl_update_latency_buckets()
2198 last_latency[rw]); in throtl_update_latency_buckets()
2199 td->avg_buckets[rw][i].valid = true; in throtl_update_latency_buckets()
2200 last_latency[rw] = td->avg_buckets[rw][i].latency; in throtl_update_latency_buckets()
2224 bool rw = bio_data_dir(bio); in blk_throtl_charge_bio_split() local
2227 if (!parent->has_rules[rw]) in blk_throtl_charge_bio_split()
2230 atomic_inc(&parent->io_split_cnt[rw]); in blk_throtl_charge_bio_split()
2231 atomic_inc(&parent->last_io_split_cnt[rw]); in blk_throtl_charge_bio_split()
2245 bool rw = bio_data_dir(bio); in blk_throtl_bio() local
2261 if (!tg->has_rules[rw]) in blk_throtl_bio()
2274 if (tg->last_low_overflow_time[rw] == 0) in blk_throtl_bio()
2275 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2279 if (sq->nr_queued[rw]) in blk_throtl_bio()
2284 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2306 throtl_trim_slice(tg, rw); in blk_throtl_bio()
2313 qn = &tg->qnode_on_parent[rw]; in blk_throtl_bio()
2322 rw == READ ? 'R' : 'W', in blk_throtl_bio()
2323 tg->bytes_disp[rw], bio->bi_iter.bi_size, in blk_throtl_bio()
2324 tg_bps_limit(tg, rw), in blk_throtl_bio()
2325 tg->io_disp[rw], tg_iops_limit(tg, rw), in blk_throtl_bio()
2328 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2330 td->nr_queued[rw]++; in blk_throtl_bio()
2395 int rw = bio_data_dir(bio); in blk_throtl_bio_endio() local
2423 threshold = tg->td->avg_buckets[rw][bucket].latency + in blk_throtl_bio_endio()