29 struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT,
69 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,  in blkg_alloc()  argument
82 	blkg->blkcg = blkcg;  in blkg_alloc()
86 	if (blkcg != &blkcg_root) {  in blkg_alloc()
127 struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q,  in __blkg_lookup()  argument
132 	blkg = rcu_dereference(blkcg->blkg_hint);  in __blkg_lookup()
142 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in __blkg_lookup()
146 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in __blkg_lookup()
163 struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q)  in blkg_lookup()  argument
169 	return __blkg_lookup(blkcg, q, false);  in blkg_lookup()
177 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,  in blkg_create()  argument
188 	if (!css_tryget_online(&blkcg->css)) {  in blkg_create()
195 		new_blkg = blkg_alloc(blkcg, q, GFP_ATOMIC);  in blkg_create()
204 	if (blkcg_parent(blkcg)) {  in blkg_create()
205 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);  in blkg_create()
222 	spin_lock(&blkcg->lock);  in blkg_create()
223 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);  in blkg_create()
225 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
236 	spin_unlock(&blkcg->lock);  in blkg_create()
239 		if (blkcg == &blkcg_root) {  in blkg_create()
251 	css_put(&blkcg->css);  in blkg_create()
271 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,  in blkg_lookup_create()  argument
286 	blkg = __blkg_lookup(blkcg, q, true);  in blkg_lookup_create()
295 		struct blkcg *pos = blkcg;  in blkg_lookup_create()
296 		struct blkcg *parent = blkcg_parent(blkcg);  in blkg_lookup_create()
304 		if (pos == blkcg || IS_ERR(blkg))  in blkg_lookup_create()
312 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()  local
316 	lockdep_assert_held(&blkcg->lock);  in blkg_destroy()
330 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
339 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
340 		rcu_assign_pointer(blkcg->blkg_hint, NULL);  in blkg_destroy()
346 	if (blkcg == &blkcg_root) {  in blkg_destroy()
371 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()  local
373 		spin_lock(&blkcg->lock);  in blkg_destroy_all()
375 		spin_unlock(&blkcg->lock);  in blkg_destroy_all()
401 	css_put(&blkg->blkcg->css);  in __blkg_release_rcu()
447 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_reset_stats()  local
465 	spin_lock_irq(&blkcg->lock);  in blkcg_reset_stats()
472 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
482 	spin_unlock_irq(&blkcg->lock);  in blkcg_reset_stats()
513 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,  in blkcg_print_blkgs()  argument
523 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
705 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,  in blkg_conf_prep()  argument
730 		blkg = blkg_lookup_create(blkcg, disk->queue);  in blkg_conf_prep()
796 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_offline()  local
798 	spin_lock_irq(&blkcg->lock);  in blkcg_css_offline()
800 	while (!hlist_empty(&blkcg->blkg_list)) {  in blkcg_css_offline()
801 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_css_offline()
809 			spin_unlock_irq(&blkcg->lock);  in blkcg_css_offline()
811 			spin_lock_irq(&blkcg->lock);  in blkcg_css_offline()
815 	spin_unlock_irq(&blkcg->lock);  in blkcg_css_offline()
820 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_free()  local
822 	if (blkcg != &blkcg_root)  in blkcg_css_free()
823 		kfree(blkcg);  in blkcg_css_free()
829 	struct blkcg *blkcg;  in blkcg_css_alloc()  local
832 		blkcg = &blkcg_root;  in blkcg_css_alloc()
836 	blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);  in blkcg_css_alloc()
837 	if (!blkcg)  in blkcg_css_alloc()
840 	blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;  in blkcg_css_alloc()
841 	blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;  in blkcg_css_alloc()
843 	spin_lock_init(&blkcg->lock);  in blkcg_css_alloc()
844 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);  in blkcg_css_alloc()
845 	INIT_HLIST_HEAD(&blkcg->blkg_list);  in blkcg_css_alloc()
847 	return &blkcg->css;  in blkcg_css_alloc()
1036 		spin_lock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1043 		spin_unlock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1085 		spin_lock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()
1095 		spin_unlock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()