46 struct blkcg blkcg_root;
96 	css_put(&blkg->blkcg->css);  in __blkg_release()
151 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,  in blkg_alloc()  argument
174 	blkg->blkcg = blkcg;  in blkg_alloc()
188 		pd = pol->pd_alloc_fn(gfp_mask, q, blkcg);  in blkg_alloc()
204 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,  in blkg_lookup_slowpath()  argument
215 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup_slowpath()
219 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in blkg_lookup_slowpath()
232 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,  in blkg_create()  argument
249 	if (!css_tryget_online(&blkcg->css)) {  in blkg_create()
256 		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);  in blkg_create()
265 	if (blkcg_parent(blkcg)) {  in blkg_create()
266 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);  in blkg_create()
283 	spin_lock(&blkcg->lock);  in blkg_create()
284 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);  in blkg_create()
286 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
297 	spin_unlock(&blkcg->lock);  in blkg_create()
307 	css_put(&blkcg->css);  in blkg_create()
326 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,  in blkg_lookup_create()  argument
334 	blkg = blkg_lookup(blkcg, q);  in blkg_lookup_create()
339 	blkg = __blkg_lookup(blkcg, q, true);  in blkg_lookup_create()
349 		struct blkcg *pos = blkcg;  in blkg_lookup_create()
350 		struct blkcg *parent = blkcg_parent(blkcg);  in blkg_lookup_create()
369 		if (pos == blkcg)  in blkg_lookup_create()
380 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()  local
384 	lockdep_assert_held(&blkcg->lock);  in blkg_destroy()
399 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
408 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
409 		rcu_assign_pointer(blkcg->blkg_hint, NULL);  in blkg_destroy()
433 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()  local
435 		spin_lock(&blkcg->lock);  in blkg_destroy_all()
437 		spin_unlock(&blkcg->lock);  in blkg_destroy_all()
470 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_reset_stats()  local
475 	spin_lock_irq(&blkcg->lock);  in blkcg_reset_stats()
482 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
498 	spin_unlock_irq(&blkcg->lock);  in blkcg_reset_stats()
528 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,  in blkcg_print_blkgs()  argument
538 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
572 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,  in blkg_lookup_check()  argument
581 	return __blkg_lookup(blkcg, q, true /* update_hint */);  in blkg_lookup_check()
634 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,  in blkg_conf_prep()  argument
660 	blkg = blkg_lookup_check(blkcg, pol, q);  in blkg_conf_prep()
674 		struct blkcg *pos = blkcg;  in blkg_conf_prep()
675 		struct blkcg *parent;  in blkg_conf_prep()
678 		parent = blkcg_parent(blkcg);  in blkg_conf_prep()
722 		if (pos == blkcg)  in blkg_conf_prep()
803 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_rstat_flush()  local
812 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_rstat_flush()
958 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));  in blkcg_print_stat()  local
964 		cgroup_rstat_flush(blkcg->css.cgroup);  in blkcg_print_stat()
967 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_stat()
1023 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_offline()  local
1026 	wb_blkcg_offline(blkcg);  in blkcg_css_offline()
1029 	blkcg_unpin_online(blkcg);  in blkcg_css_offline()
1043 void blkcg_destroy_blkgs(struct blkcg *blkcg)  in blkcg_destroy_blkgs()  argument
1047 	spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1049 	while (!hlist_empty(&blkcg->blkg_list)) {  in blkcg_destroy_blkgs()
1050 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_destroy_blkgs()
1060 			spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1062 			spin_lock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1070 	spin_unlock_irq(&blkcg->lock);  in blkcg_destroy_blkgs()
1075 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_free()  local
1080 	list_del(&blkcg->all_blkcgs_node);  in blkcg_css_free()
1083 		if (blkcg->cpd[i])  in blkcg_css_free()
1084 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_free()
1088 	kfree(blkcg);  in blkcg_css_free()
1094 	struct blkcg *blkcg;  in blkcg_css_alloc()  local
1101 		blkcg = &blkcg_root;  in blkcg_css_alloc()
1103 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);  in blkcg_css_alloc()
1104 		if (!blkcg) {  in blkcg_css_alloc()
1128 		blkcg->cpd[i] = cpd;  in blkcg_css_alloc()
1129 		cpd->blkcg = blkcg;  in blkcg_css_alloc()
1135 	spin_lock_init(&blkcg->lock);  in blkcg_css_alloc()
1136 	refcount_set(&blkcg->online_pin, 1);  in blkcg_css_alloc()
1137 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);  in blkcg_css_alloc()
1138 	INIT_HLIST_HEAD(&blkcg->blkg_list);  in blkcg_css_alloc()
1140 	INIT_LIST_HEAD(&blkcg->cgwb_list);  in blkcg_css_alloc()
1142 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);  in blkcg_css_alloc()
1145 	return &blkcg->css;  in blkcg_css_alloc()
1149 		if (blkcg->cpd[i])  in blkcg_css_alloc()
1150 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_alloc()
1152 	if (blkcg != &blkcg_root)  in blkcg_css_alloc()
1153 		kfree(blkcg);  in blkcg_css_alloc()
1161 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_online()  local
1162 	struct blkcg *parent = blkcg_parent(blkcg);  in blkcg_css_online()
1256 		struct blkcg *blkcg;  in blkcg_bind()  local
1261 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)  in blkcg_bind()
1262 			if (blkcg->cpd[pol->plid])  in blkcg_bind()
1263 				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);  in blkcg_bind()
1341 					      blkg->blkcg);  in blkcg_activate_policy()
1359 						       blkg->blkcg);  in blkcg_activate_policy()
1397 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_activate_policy()  local
1399 		spin_lock(&blkcg->lock);  in blkcg_activate_policy()
1404 		spin_unlock(&blkcg->lock);  in blkcg_activate_policy()
1436 		struct blkcg *blkcg = blkg->blkcg;  in blkcg_deactivate_policy()  local
1438 		spin_lock(&blkcg->lock);  in blkcg_deactivate_policy()
1445 		spin_unlock(&blkcg->lock);  in blkcg_deactivate_policy()
1464 	struct blkcg *blkcg;  in blkcg_policy_register()  local
1491 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1498 			blkcg->cpd[pol->plid] = cpd;  in blkcg_policy_register()
1499 			cpd->blkcg = blkcg;  in blkcg_policy_register()
1520 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1521 			if (blkcg->cpd[pol->plid]) {  in blkcg_policy_register()
1522 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_policy_register()
1523 				blkcg->cpd[pol->plid] = NULL;  in blkcg_policy_register()
1543 	struct blkcg *blkcg;  in blkcg_policy_unregister()  local
1560 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_unregister()
1561 			if (blkcg->cpd[pol->plid]) {  in blkcg_policy_unregister()
1562 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_policy_unregister()
1563 				blkcg->cpd[pol->plid] = NULL;  in blkcg_policy_unregister()
1727 	struct blkcg *blkcg;  in blkcg_maybe_throttle_current()  local
1740 		blkcg = css_to_blkcg(css);  in blkcg_maybe_throttle_current()
1742 		blkcg = css_to_blkcg(task_css(current, io_cgrp_id));  in blkcg_maybe_throttle_current()
1744 	if (!blkcg)  in blkcg_maybe_throttle_current()
1746 	blkg = blkg_lookup(blkcg, q);  in blkcg_maybe_throttle_current()
1942 		cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);  in blk_cgroup_bio_start()