71 	struct throtl_grp	*tg;		/* tg this qnode belongs to */
99 #define rb_entry_tg(node)	rb_entry((node), struct throtl_grp, rb_node)
107 struct throtl_grp {  struct
234 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)  in pd_to_tg()
236 	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;  in pd_to_tg()
239 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)  in blkg_to_tg()
244 static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)  in tg_to_blkg()
256 static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)  in sq_to_tg()
259 		return container_of(sq, struct throtl_grp, service_queue);  in sq_to_tg()
273 	struct throtl_grp *tg = sq_to_tg(sq);  in sq_to_td()
299 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)  in tg_bps_limit()
329 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)  in tg_iops_limit()
374 	struct throtl_grp *__tg = sq_to_tg((sq));			\
396 static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg)  in throtl_qnode_init()
456 				     struct throtl_grp **tg_to_put)  in throtl_pop_queued()
494 	struct throtl_grp *tg;  in throtl_pd_alloc()
541 	struct throtl_grp *tg = pd_to_tg(pd);  in throtl_pd_init()
570 static void tg_update_has_rules(struct throtl_grp *tg)  in tg_update_has_rules()
572 	struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq);  in tg_update_has_rules()
585 	struct throtl_grp *tg = pd_to_tg(pd);  in throtl_pd_online()
601 		struct throtl_grp *tg = blkg_to_tg(blkg);  in blk_throtl_update_limit_valid()
617 	struct throtl_grp *tg = pd_to_tg(pd);  in throtl_pd_offline()
632 	struct throtl_grp *tg = pd_to_tg(pd);  in throtl_pd_free()
640 static struct throtl_grp *
662 	struct throtl_grp *tg;  in update_min_dispatch_time()
671 static void tg_service_queue_add(struct throtl_grp *tg)  in tg_service_queue_add()
676 	struct throtl_grp *__tg;  in tg_service_queue_add()
697 static void throtl_enqueue_tg(struct throtl_grp *tg)  in throtl_enqueue_tg()
706 static void throtl_dequeue_tg(struct throtl_grp *tg)  in throtl_dequeue_tg()
771 static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg,  in throtl_start_new_slice_with_credit()
795 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)  in throtl_start_new_slice()
810 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,  in throtl_set_slice_end()
816 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,  in throtl_extend_slice()
827 static bool throtl_slice_used(struct throtl_grp *tg, bool rw)  in throtl_slice_used()
836 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)  in throtl_trim_slice()
895 static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_iops_limit()
943 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_bps_limit()
994 static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,  in tg_may_dispatch()
1055 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)  in throtl_charge_bio()
1086 			      struct throtl_grp *tg)  in throtl_add_bio_tg()
1109 static void tg_update_disptime(struct throtl_grp *tg)  in tg_update_disptime()
1135 static void start_parent_slice_with_credit(struct throtl_grp *child_tg,  in start_parent_slice_with_credit()
1136 					struct throtl_grp *parent_tg, bool rw)  in start_parent_slice_with_credit()
1145 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)  in tg_dispatch_one_bio()
1149 	struct throtl_grp *parent_tg = sq_to_tg(parent_sq);  in tg_dispatch_one_bio()
1150 	struct throtl_grp *tg_to_put = NULL;  in tg_dispatch_one_bio()
1187 static int throtl_dispatch_tg(struct throtl_grp *tg)  in throtl_dispatch_tg()
1225 		struct throtl_grp *tg;  in throtl_select_dispatch()
1254 	struct throtl_grp *this_tg);
1273 	struct throtl_grp *tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1368 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_conf_u64()
1379 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_conf_uint()
1401 static void tg_conf_updated(struct throtl_grp *tg, bool global)  in tg_conf_updated()
1422 		struct throtl_grp *this_tg = blkg_to_tg(blkg);  in tg_conf_updated()
1423 		struct throtl_grp *parent_tg;  in tg_conf_updated()
1464 	struct throtl_grp *tg;  in tg_set_conf()
1533 		.private = offsetof(struct throtl_grp, bps[READ][LIMIT_MAX]),
1539 		.private = offsetof(struct throtl_grp, bps[WRITE][LIMIT_MAX]),
1545 		.private = offsetof(struct throtl_grp, iops[READ][LIMIT_MAX]),
1551 		.private = offsetof(struct throtl_grp, iops[WRITE][LIMIT_MAX]),
1557 		.private = offsetof(struct throtl_grp, stat_bytes),
1562 		.private = offsetof(struct throtl_grp, stat_bytes),
1567 		.private = offsetof(struct throtl_grp, stat_ios),
1572 		.private = offsetof(struct throtl_grp, stat_ios),
1581 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_limit()
1653 	struct throtl_grp *tg;  in tg_set_limit()
1802 static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg)  in __tg_last_low_overflow_time()
1814 static unsigned long tg_last_low_overflow_time(struct throtl_grp *tg)  in tg_last_low_overflow_time()
1817 	struct throtl_grp *parent = tg;  in tg_last_low_overflow_time()
1841 static bool throtl_tg_is_idle(struct throtl_grp *tg)  in throtl_tg_is_idle()
1867 static bool throtl_tg_can_upgrade(struct throtl_grp *tg)  in throtl_tg_can_upgrade()
1894 static bool throtl_hierarchy_can_upgrade(struct throtl_grp *tg)  in throtl_hierarchy_can_upgrade()
1907 	struct throtl_grp *this_tg)  in throtl_can_upgrade()
1920 		struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_can_upgrade()
1935 static void throtl_upgrade_check(struct throtl_grp *tg)  in throtl_upgrade_check()
1966 		struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_upgrade_state()
1993 static bool throtl_tg_can_downgrade(struct throtl_grp *tg)  in throtl_tg_can_downgrade()
2011 static bool throtl_hierarchy_can_downgrade(struct throtl_grp *tg)  in throtl_hierarchy_can_downgrade()
2023 static void throtl_downgrade_check(struct throtl_grp *tg)  in throtl_downgrade_check()
2086 static void blk_throtl_update_idletime(struct throtl_grp *tg)  in blk_throtl_update_idletime()
2189 	struct throtl_grp *parent = blkg_to_tg(blkg);  in blk_throtl_charge_bio_split()
2210 	struct throtl_grp *tg = blkg_to_tg(blkg);  in blk_throtl_bio()
2357 	struct throtl_grp *tg;  in blk_throtl_bio_endio()