Lines Matching refs:blkcg
47 struct blkcg blkcg_root;
95 css_put(&blkg->blkcg->css); in __blkg_release()
150 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, in blkg_alloc() argument
173 blkg->blkcg = blkcg; in blkg_alloc()
187 pd = pol->pd_alloc_fn(gfp_mask, q, blkcg); in blkg_alloc()
203 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, in blkg_lookup_slowpath() argument
214 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); in blkg_lookup_slowpath()
218 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_slowpath()
231 static struct blkcg_gq *blkg_create(struct blkcg *blkcg, in blkg_create() argument
248 if (!css_tryget_online(&blkcg->css)) { in blkg_create()
255 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN); in blkg_create()
264 if (blkcg_parent(blkcg)) { in blkg_create()
265 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); in blkg_create()
282 spin_lock(&blkcg->lock); in blkg_create()
283 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); in blkg_create()
285 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
296 spin_unlock(&blkcg->lock); in blkg_create()
306 css_put(&blkcg->css); in blkg_create()
325 static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, in blkg_lookup_create() argument
333 blkg = blkg_lookup(blkcg, q); in blkg_lookup_create()
338 blkg = __blkg_lookup(blkcg, q, true); in blkg_lookup_create()
348 struct blkcg *pos = blkcg; in blkg_lookup_create()
349 struct blkcg *parent = blkcg_parent(blkcg); in blkg_lookup_create()
368 if (pos == blkcg) in blkg_lookup_create()
379 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy() local
383 lockdep_assert_held(&blkcg->lock); in blkg_destroy()
398 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
407 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
408 rcu_assign_pointer(blkcg->blkg_hint, NULL); in blkg_destroy()
429 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all() local
431 spin_lock(&blkcg->lock); in blkg_destroy_all()
433 spin_unlock(&blkcg->lock); in blkg_destroy_all()
443 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_reset_stats() local
448 spin_lock_irq(&blkcg->lock); in blkcg_reset_stats()
455 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
471 spin_unlock_irq(&blkcg->lock); in blkcg_reset_stats()
502 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, in blkcg_print_blkgs() argument
512 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
546 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg, in blkg_lookup_check() argument
555 return __blkg_lookup(blkcg, q, true /* update_hint */); in blkg_lookup_check()
608 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, in blkg_conf_prep() argument
634 blkg = blkg_lookup_check(blkcg, pol, q); in blkg_conf_prep()
648 struct blkcg *pos = blkcg; in blkg_conf_prep()
649 struct blkcg *parent; in blkg_conf_prep()
652 parent = blkcg_parent(blkcg); in blkg_conf_prep()
696 if (pos == blkcg) in blkg_conf_prep()
777 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_rstat_flush() local
782 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_rstat_flush()
869 struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); in blkcg_print_stat() local
875 cgroup_rstat_flush(blkcg->css.cgroup); in blkcg_print_stat()
879 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
1009 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_offline() local
1012 wb_blkcg_offline(blkcg); in blkcg_css_offline()
1015 blkcg_unpin_online(blkcg); in blkcg_css_offline()
1029 void blkcg_destroy_blkgs(struct blkcg *blkcg) in blkcg_destroy_blkgs() argument
1033 spin_lock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1035 while (!hlist_empty(&blkcg->blkg_list)) { in blkcg_destroy_blkgs()
1036 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_destroy_blkgs()
1046 spin_unlock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1048 spin_lock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1056 spin_unlock_irq(&blkcg->lock); in blkcg_destroy_blkgs()
1061 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_free() local
1066 list_del(&blkcg->all_blkcgs_node); in blkcg_css_free()
1069 if (blkcg->cpd[i]) in blkcg_css_free()
1070 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); in blkcg_css_free()
1074 kfree(blkcg); in blkcg_css_free()
1080 struct blkcg *blkcg; in blkcg_css_alloc() local
1087 blkcg = &blkcg_root; in blkcg_css_alloc()
1089 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); in blkcg_css_alloc()
1090 if (!blkcg) { in blkcg_css_alloc()
1114 blkcg->cpd[i] = cpd; in blkcg_css_alloc()
1115 cpd->blkcg = blkcg; in blkcg_css_alloc()
1121 spin_lock_init(&blkcg->lock); in blkcg_css_alloc()
1122 refcount_set(&blkcg->online_pin, 1); in blkcg_css_alloc()
1123 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN); in blkcg_css_alloc()
1124 INIT_HLIST_HEAD(&blkcg->blkg_list); in blkcg_css_alloc()
1126 INIT_LIST_HEAD(&blkcg->cgwb_list); in blkcg_css_alloc()
1128 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs); in blkcg_css_alloc()
1131 return &blkcg->css; in blkcg_css_alloc()
1135 if (blkcg->cpd[i]) in blkcg_css_alloc()
1136 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); in blkcg_css_alloc()
1138 if (blkcg != &blkcg_root) in blkcg_css_alloc()
1139 kfree(blkcg); in blkcg_css_alloc()
1147 struct blkcg *blkcg = css_to_blkcg(css); in blkcg_css_online() local
1148 struct blkcg *parent = blkcg_parent(blkcg); in blkcg_css_online()
1263 struct blkcg *blkcg; in blkcg_bind() local
1268 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) in blkcg_bind()
1269 if (blkcg->cpd[pol->plid]) in blkcg_bind()
1270 pol->cpd_bind_fn(blkcg->cpd[pol->plid]); in blkcg_bind()
1349 blkg->blkcg); in blkcg_activate_policy()
1367 blkg->blkcg); in blkcg_activate_policy()
1405 struct blkcg *blkcg = blkg->blkcg; in blkcg_activate_policy() local
1407 spin_lock(&blkcg->lock); in blkcg_activate_policy()
1412 spin_unlock(&blkcg->lock); in blkcg_activate_policy()
1444 struct blkcg *blkcg = blkg->blkcg; in blkcg_deactivate_policy() local
1446 spin_lock(&blkcg->lock); in blkcg_deactivate_policy()
1453 spin_unlock(&blkcg->lock); in blkcg_deactivate_policy()
1472 struct blkcg *blkcg; in blkcg_policy_register() local
1499 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_register()
1506 blkcg->cpd[pol->plid] = cpd; in blkcg_policy_register()
1507 cpd->blkcg = blkcg; in blkcg_policy_register()
1528 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_register()
1529 if (blkcg->cpd[pol->plid]) { in blkcg_policy_register()
1530 pol->cpd_free_fn(blkcg->cpd[pol->plid]); in blkcg_policy_register()
1531 blkcg->cpd[pol->plid] = NULL; in blkcg_policy_register()
1551 struct blkcg *blkcg; in blkcg_policy_unregister() local
1568 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { in blkcg_policy_unregister()
1569 if (blkcg->cpd[pol->plid]) { in blkcg_policy_unregister()
1570 pol->cpd_free_fn(blkcg->cpd[pol->plid]); in blkcg_policy_unregister()
1571 blkcg->cpd[pol->plid] = NULL; in blkcg_policy_unregister()
1735 struct blkcg *blkcg; in blkcg_maybe_throttle_current() local
1748 blkcg = css_to_blkcg(css); in blkcg_maybe_throttle_current()
1750 blkcg = css_to_blkcg(task_css(current, io_cgrp_id)); in blkcg_maybe_throttle_current()
1752 if (!blkcg) in blkcg_maybe_throttle_current()
1754 blkg = blkg_lookup(blkcg, q); in blkcg_maybe_throttle_current()
1945 cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu); in blk_cgroup_bio_start()