Lines Matching refs:blkg
64 static void blkg_free(struct blkcg_gq *blkg) in blkg_free() argument
68 if (!blkg) in blkg_free()
72 if (blkg->pd[i]) in blkg_free()
73 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); in blkg_free()
75 if (blkg->blkcg != &blkcg_root) in blkg_free()
76 blk_exit_rl(&blkg->rl); in blkg_free()
78 blkg_rwstat_exit(&blkg->stat_ios); in blkg_free()
79 blkg_rwstat_exit(&blkg->stat_bytes); in blkg_free()
80 kfree(blkg); in blkg_free()
94 struct blkcg_gq *blkg; in blkg_alloc() local
98 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); in blkg_alloc()
99 if (!blkg) in blkg_alloc()
102 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) || in blkg_alloc()
103 blkg_rwstat_init(&blkg->stat_ios, gfp_mask)) in blkg_alloc()
106 blkg->q = q; in blkg_alloc()
107 INIT_LIST_HEAD(&blkg->q_node); in blkg_alloc()
108 blkg->blkcg = blkcg; in blkg_alloc()
109 atomic_set(&blkg->refcnt, 1); in blkg_alloc()
113 if (blk_init_rl(&blkg->rl, q, gfp_mask)) in blkg_alloc()
115 blkg->rl.blkg = blkg; in blkg_alloc()
130 blkg->pd[i] = pd; in blkg_alloc()
131 pd->blkg = blkg; in blkg_alloc()
135 return blkg; in blkg_alloc()
138 blkg_free(blkg); in blkg_alloc()
145 struct blkcg_gq *blkg; in blkg_lookup_slowpath() local
153 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); in blkg_lookup_slowpath()
154 if (blkg && blkg->q == q) { in blkg_lookup_slowpath()
157 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_slowpath()
159 return blkg; in blkg_lookup_slowpath()
174 struct blkcg_gq *blkg; in blkg_create() local
203 blkg = new_blkg; in blkg_create()
204 blkg->wb_congested = wb_congested; in blkg_create()
208 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); in blkg_create()
209 if (WARN_ON_ONCE(!blkg->parent)) { in blkg_create()
213 blkg_get(blkg->parent); in blkg_create()
220 if (blkg->pd[i] && pol->pd_init_fn) in blkg_create()
221 pol->pd_init_fn(blkg->pd[i]); in blkg_create()
226 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); in blkg_create()
228 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
229 list_add(&blkg->q_node, &q->blkg_list); in blkg_create()
234 if (blkg->pd[i] && pol->pd_online_fn) in blkg_create()
235 pol->pd_online_fn(blkg->pd[i]); in blkg_create()
238 blkg->online = true; in blkg_create()
242 return blkg; in blkg_create()
245 blkg_put(blkg); in blkg_create()
274 struct blkcg_gq *blkg; in blkg_lookup_create() local
286 blkg = __blkg_lookup(blkcg, q, true); in blkg_lookup_create()
287 if (blkg) in blkg_lookup_create()
288 return blkg; in blkg_lookup_create()
303 blkg = blkg_create(pos, q, NULL); in blkg_lookup_create()
304 if (pos == blkcg || IS_ERR(blkg)) in blkg_lookup_create()
305 return blkg; in blkg_lookup_create()
309 static void blkg_destroy(struct blkcg_gq *blkg) in blkg_destroy() argument
311 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy()
312 struct blkcg_gq *parent = blkg->parent; in blkg_destroy()
315 lockdep_assert_held(blkg->q->queue_lock); in blkg_destroy()
319 WARN_ON_ONCE(list_empty(&blkg->q_node)); in blkg_destroy()
320 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); in blkg_destroy()
325 if (blkg->pd[i] && pol->pd_offline_fn) in blkg_destroy()
326 pol->pd_offline_fn(blkg->pd[i]); in blkg_destroy()
330 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes); in blkg_destroy()
331 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios); in blkg_destroy()
334 blkg->online = false; in blkg_destroy()
336 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
337 list_del_init(&blkg->q_node); in blkg_destroy()
338 hlist_del_init_rcu(&blkg->blkcg_node); in blkg_destroy()
345 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
352 blkg_put(blkg); in blkg_destroy()
363 struct blkcg_gq *blkg, *n; in blkg_destroy_all() local
367 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { in blkg_destroy_all()
368 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all()
371 blkg_destroy(blkg); in blkg_destroy_all()
376 q->root_rl.blkg = NULL; in blkg_destroy_all()
389 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); in __blkg_release_rcu() local
392 css_put(&blkg->blkcg->css); in __blkg_release_rcu()
393 if (blkg->parent) in __blkg_release_rcu()
394 blkg_put(blkg->parent); in __blkg_release_rcu()
396 wb_congested_put(blkg->wb_congested); in __blkg_release_rcu()
398 blkg_free(blkg); in __blkg_release_rcu()
410 struct blkcg_gq *blkg; in __blk_queue_next_rl() local
422 blkg = container_of(rl, struct blkcg_gq, rl); in __blk_queue_next_rl()
423 ent = &blkg->q_node; in __blk_queue_next_rl()
433 blkg = container_of(ent, struct blkcg_gq, q_node); in __blk_queue_next_rl()
434 return &blkg->rl; in __blk_queue_next_rl()
441 struct blkcg_gq *blkg; in blkcg_reset_stats() local
452 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
453 blkg_rwstat_reset(&blkg->stat_bytes); in blkcg_reset_stats()
454 blkg_rwstat_reset(&blkg->stat_ios); in blkcg_reset_stats()
459 if (blkg->pd[i] && pol->pd_reset_stats_fn) in blkcg_reset_stats()
460 pol->pd_reset_stats_fn(blkg->pd[i]); in blkcg_reset_stats()
469 const char *blkg_dev_name(struct blkcg_gq *blkg) in blkg_dev_name() argument
472 if (blkg->q->backing_dev_info.dev) in blkg_dev_name()
473 return dev_name(blkg->q->backing_dev_info.dev); in blkg_dev_name()
502 struct blkcg_gq *blkg; in blkcg_print_blkgs() local
506 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
507 spin_lock_irq(blkg->q->queue_lock); in blkcg_print_blkgs()
508 if (blkcg_policy_enabled(blkg->q, pol)) in blkcg_print_blkgs()
509 total += prfill(sf, blkg->pd[pol->plid], data); in blkcg_print_blkgs()
510 spin_unlock_irq(blkg->q->queue_lock); in blkcg_print_blkgs()
529 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_u64()
556 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_rwstat()
608 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off); in blkg_prfill_rwstat_field()
651 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg, in blkg_prfill_rwstat_field_recursive()
699 u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg, in blkg_stat_recursive_sum() argument
706 lockdep_assert_held(blkg->q->queue_lock); in blkg_stat_recursive_sum()
709 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { in blkg_stat_recursive_sum()
718 stat = (void *)blkg + off; in blkg_stat_recursive_sum()
741 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, in blkg_rwstat_recursive_sum() argument
749 lockdep_assert_held(blkg->q->queue_lock); in blkg_rwstat_recursive_sum()
752 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { in blkg_rwstat_recursive_sum()
791 struct blkcg_gq *blkg; in blkg_conf_prep() local
819 blkg = blkg_lookup_create(blkcg, disk->queue); in blkg_conf_prep()
821 blkg = ERR_PTR(-EOPNOTSUPP); in blkg_conf_prep()
823 if (IS_ERR(blkg)) { in blkg_conf_prep()
824 ret = PTR_ERR(blkg); in blkg_conf_prep()
844 ctx->blkg = blkg; in blkg_conf_prep()
873 struct blkcg_gq *blkg; in blkcg_print_stat() local
877 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
882 dname = blkg_dev_name(blkg); in blkcg_print_stat()
886 spin_lock_irq(blkg->q->queue_lock); in blkcg_print_stat()
888 rwstat = blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
893 rwstat = blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
898 spin_unlock_irq(blkg->q->queue_lock); in blkcg_print_stat()
944 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_css_offline() local
946 struct request_queue *q = blkg->q; in blkcg_css_offline()
949 blkg_destroy(blkg); in blkcg_css_offline()
1058 struct blkcg_gq *new_blkg, *blkg; in blkcg_init_queue() local
1075 blkg = blkg_create(&blkcg_root, q, new_blkg); in blkcg_init_queue()
1082 if (IS_ERR(blkg)) in blkcg_init_queue()
1083 return PTR_ERR(blkg); in blkcg_init_queue()
1085 q->root_blkg = blkg; in blkcg_init_queue()
1086 q->root_rl.blkg = blkg; in blkcg_init_queue()
1218 struct blkcg_gq *blkg; in blkcg_activate_policy() local
1236 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1239 if (blkg->pd[pol->plid]) in blkcg_activate_policy()
1250 blkg->pd[pol->plid] = pd; in blkcg_activate_policy()
1251 pd->blkg = blkg; in blkcg_activate_policy()
1280 struct blkcg_gq *blkg; in blkcg_deactivate_policy() local
1290 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_deactivate_policy()
1292 spin_lock(&blkg->blkcg->lock); in blkcg_deactivate_policy()
1294 if (blkg->pd[pol->plid]) { in blkcg_deactivate_policy()
1296 pol->pd_offline_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1297 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1298 blkg->pd[pol->plid] = NULL; in blkcg_deactivate_policy()
1301 spin_unlock(&blkg->blkcg->lock); in blkcg_deactivate_policy()