181 static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)  in sq_to_tg()  argument
183 	if (sq && sq->parent_sq)  in sq_to_tg()
184 		return container_of(sq, struct throtl_grp, service_queue);  in sq_to_tg()
196 static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)  in sq_to_td()  argument
198 	struct throtl_grp *tg = sq_to_tg(sq);  in sq_to_td()
203 		return container_of(sq, struct throtl_data, service_queue);  in sq_to_td()
218 #define throtl_log(sq, fmt, args...)	do {				\  argument
219 	struct throtl_grp *__tg = sq_to_tg((sq));			\
220 	struct throtl_data *__td = sq_to_td((sq));			\
317 static void throtl_service_queue_init(struct throtl_service_queue *sq)  in throtl_service_queue_init()  argument
319 	INIT_LIST_HEAD(&sq->queued[0]);  in throtl_service_queue_init()
320 	INIT_LIST_HEAD(&sq->queued[1]);  in throtl_service_queue_init()
321 	sq->pending_tree = RB_ROOT;  in throtl_service_queue_init()
322 	setup_timer(&sq->pending_timer, throtl_pending_timer_fn,  in throtl_service_queue_init()
323 		    (unsigned long)sq);  in throtl_service_queue_init()
356 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_pd_init()  local
371 	sq->parent_sq = &td->service_queue;  in throtl_pd_init()
373 		sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue;  in throtl_pd_init()
505 static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,  in throtl_schedule_pending_timer()  argument
519 	mod_timer(&sq->pending_timer, expires);  in throtl_schedule_pending_timer()
520 	throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",  in throtl_schedule_pending_timer()
542 static bool throtl_schedule_next_dispatch(struct throtl_service_queue *sq,  in throtl_schedule_next_dispatch()  argument
546 	if (!sq->nr_pending)  in throtl_schedule_next_dispatch()
549 	update_min_dispatch_time(sq);  in throtl_schedule_next_dispatch()
552 	if (force || time_after(sq->first_pending_disptime, jiffies)) {  in throtl_schedule_next_dispatch()
553 		throtl_schedule_pending_timer(sq, sq->first_pending_disptime);  in throtl_schedule_next_dispatch()
857 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_add_bio_tg()  local
869 	if (!sq->nr_queued[rw])  in throtl_add_bio_tg()
872 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
874 	sq->nr_queued[rw]++;  in throtl_add_bio_tg()
880 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_update_disptime()  local
884 	if ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_update_disptime()
887 	if ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_update_disptime()
914 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_dispatch_one_bio()  local
915 	struct throtl_service_queue *parent_sq = sq->parent_sq;  in tg_dispatch_one_bio()
926 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
927 	sq->nr_queued[rw]--;  in tg_dispatch_one_bio()
956 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_dispatch_tg()  local
964 	while ((bio = throtl_peek_queued(&sq->queued[READ])) &&  in throtl_dispatch_tg()
974 	while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&  in throtl_dispatch_tg()
993 		struct throtl_service_queue *sq = &tg->service_queue;  in throtl_select_dispatch()  local
1005 		if (sq->nr_queued[0] || sq->nr_queued[1])  in throtl_select_dispatch()
1032 	struct throtl_service_queue *sq = (void *)arg;  in throtl_pending_timer_fn()  local
1033 	struct throtl_grp *tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1034 	struct throtl_data *td = sq_to_td(sq);  in throtl_pending_timer_fn()
1042 	parent_sq = sq->parent_sq;  in throtl_pending_timer_fn()
1046 		throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",  in throtl_pending_timer_fn()
1047 			   sq->nr_queued[READ] + sq->nr_queued[WRITE],  in throtl_pending_timer_fn()
1048 			   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in throtl_pending_timer_fn()
1050 		ret = throtl_select_dispatch(sq);  in throtl_pending_timer_fn()
1052 			throtl_log(sq, "bios disp=%u", ret);  in throtl_pending_timer_fn()
1056 		if (throtl_schedule_next_dispatch(sq, false))  in throtl_pending_timer_fn()
1074 				sq = parent_sq;  in throtl_pending_timer_fn()
1075 				tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1160 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_conf_updated()  local
1192 		throtl_schedule_next_dispatch(sq->parent_sq, true);  in tg_conf_updated()
1412 	struct throtl_service_queue *sq;  in blk_throtl_bio()  local
1427 	sq = &tg->service_queue;  in blk_throtl_bio()
1431 		if (sq->nr_queued[rw])  in blk_throtl_bio()
1460 		sq = sq->parent_sq;  in blk_throtl_bio()
1461 		tg = sq_to_tg(sq);  in blk_throtl_bio()
1467 	throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",  in blk_throtl_bio()
1471 		   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in blk_throtl_bio()
1512 		struct throtl_service_queue *sq = &tg->service_queue;  in tg_drain_bios()  local
1517 		while ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_drain_bios()
1519 		while ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_drain_bios()