Lines Matching refs:blkg
65 static void blkg_free(struct blkcg_gq *blkg) in blkg_free() argument
69 if (!blkg) in blkg_free()
73 if (blkg->pd[i]) in blkg_free()
74 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); in blkg_free()
76 if (blkg->blkcg != &blkcg_root) in blkg_free()
77 blk_exit_rl(blkg->q, &blkg->rl); in blkg_free()
79 blkg_rwstat_exit(&blkg->stat_ios); in blkg_free()
80 blkg_rwstat_exit(&blkg->stat_bytes); in blkg_free()
81 kfree(blkg); in blkg_free()
95 struct blkcg_gq *blkg; in blkg_alloc() local
99 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); in blkg_alloc()
100 if (!blkg) in blkg_alloc()
103 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) || in blkg_alloc()
104 blkg_rwstat_init(&blkg->stat_ios, gfp_mask)) in blkg_alloc()
107 blkg->q = q; in blkg_alloc()
108 INIT_LIST_HEAD(&blkg->q_node); in blkg_alloc()
109 blkg->blkcg = blkcg; in blkg_alloc()
110 atomic_set(&blkg->refcnt, 1); in blkg_alloc()
114 if (blk_init_rl(&blkg->rl, q, gfp_mask)) in blkg_alloc()
116 blkg->rl.blkg = blkg; in blkg_alloc()
131 blkg->pd[i] = pd; in blkg_alloc()
132 pd->blkg = blkg; in blkg_alloc()
136 return blkg; in blkg_alloc()
139 blkg_free(blkg); in blkg_alloc()
146 struct blkcg_gq *blkg; in blkg_lookup_slowpath() local
154 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); in blkg_lookup_slowpath()
155 if (blkg && blkg->q == q) { in blkg_lookup_slowpath()
158 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_slowpath()
160 return blkg; in blkg_lookup_slowpath()
175 struct blkcg_gq *blkg; in blkg_create() local
204 blkg = new_blkg; in blkg_create()
205 blkg->wb_congested = wb_congested; in blkg_create()
209 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); in blkg_create()
210 if (WARN_ON_ONCE(!blkg->parent)) { in blkg_create()
214 blkg_get(blkg->parent); in blkg_create()
221 if (blkg->pd[i] && pol->pd_init_fn) in blkg_create()
222 pol->pd_init_fn(blkg->pd[i]); in blkg_create()
227 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); in blkg_create()
229 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
230 list_add(&blkg->q_node, &q->blkg_list); in blkg_create()
235 if (blkg->pd[i] && pol->pd_online_fn) in blkg_create()
236 pol->pd_online_fn(blkg->pd[i]); in blkg_create()
239 blkg->online = true; in blkg_create()
243 return blkg; in blkg_create()
246 blkg_put(blkg); in blkg_create()
275 struct blkcg_gq *blkg; in blkg_lookup_create() local
287 blkg = __blkg_lookup(blkcg, q, true); in blkg_lookup_create()
288 if (blkg) in blkg_lookup_create()
289 return blkg; in blkg_lookup_create()
304 blkg = blkg_create(pos, q, NULL); in blkg_lookup_create()
305 if (pos == blkcg || IS_ERR(blkg)) in blkg_lookup_create()
306 return blkg; in blkg_lookup_create()
310 static void blkg_destroy(struct blkcg_gq *blkg) in blkg_destroy() argument
312 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy()
313 struct blkcg_gq *parent = blkg->parent; in blkg_destroy()
316 lockdep_assert_held(blkg->q->queue_lock); in blkg_destroy()
320 WARN_ON_ONCE(list_empty(&blkg->q_node)); in blkg_destroy()
321 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); in blkg_destroy()
326 if (blkg->pd[i] && pol->pd_offline_fn) in blkg_destroy()
327 pol->pd_offline_fn(blkg->pd[i]); in blkg_destroy()
331 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes); in blkg_destroy()
332 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios); in blkg_destroy()
335 blkg->online = false; in blkg_destroy()
337 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
338 list_del_init(&blkg->q_node); in blkg_destroy()
339 hlist_del_init_rcu(&blkg->blkcg_node); in blkg_destroy()
346 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
353 blkg_put(blkg); in blkg_destroy()
364 struct blkcg_gq *blkg, *n; in blkg_destroy_all() local
368 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { in blkg_destroy_all()
369 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all()
372 blkg_destroy(blkg); in blkg_destroy_all()
377 q->root_rl.blkg = NULL; in blkg_destroy_all()
390 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); in __blkg_release_rcu() local
393 css_put(&blkg->blkcg->css); in __blkg_release_rcu()
394 if (blkg->parent) in __blkg_release_rcu()
395 blkg_put(blkg->parent); in __blkg_release_rcu()
397 wb_congested_put(blkg->wb_congested); in __blkg_release_rcu()
399 blkg_free(blkg); in __blkg_release_rcu()
411 struct blkcg_gq *blkg; in __blk_queue_next_rl() local
423 blkg = container_of(rl, struct blkcg_gq, rl); in __blk_queue_next_rl()
424 ent = &blkg->q_node; in __blk_queue_next_rl()
434 blkg = container_of(ent, struct blkcg_gq, q_node); in __blk_queue_next_rl()
435 return &blkg->rl; in __blk_queue_next_rl()
442 struct blkcg_gq *blkg; in blkcg_reset_stats() local
453 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
454 blkg_rwstat_reset(&blkg->stat_bytes); in blkcg_reset_stats()
455 blkg_rwstat_reset(&blkg->stat_ios); in blkcg_reset_stats()
460 if (blkg->pd[i] && pol->pd_reset_stats_fn) in blkcg_reset_stats()
461 pol->pd_reset_stats_fn(blkg->pd[i]); in blkcg_reset_stats()
470 const char *blkg_dev_name(struct blkcg_gq *blkg) in blkg_dev_name() argument
473 if (blkg->q->backing_dev_info->dev) in blkg_dev_name()
474 return dev_name(blkg->q->backing_dev_info->dev); in blkg_dev_name()
503 struct blkcg_gq *blkg; in blkcg_print_blkgs() local
507 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
508 spin_lock_irq(blkg->q->queue_lock); in blkcg_print_blkgs()
509 if (blkcg_policy_enabled(blkg->q, pol)) in blkcg_print_blkgs()
510 total += prfill(sf, blkg->pd[pol->plid], data); in blkcg_print_blkgs()
511 spin_unlock_irq(blkg->q->queue_lock); in blkcg_print_blkgs()
530 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_u64()
557 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_rwstat()
609 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off); in blkg_prfill_rwstat_field()
652 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg, in blkg_prfill_rwstat_field_recursive()
700 u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg, in blkg_stat_recursive_sum() argument
707 lockdep_assert_held(blkg->q->queue_lock); in blkg_stat_recursive_sum()
710 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { in blkg_stat_recursive_sum()
719 stat = (void *)blkg + off; in blkg_stat_recursive_sum()
742 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, in blkg_rwstat_recursive_sum() argument
750 lockdep_assert_held(blkg->q->queue_lock); in blkg_rwstat_recursive_sum()
753 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { in blkg_rwstat_recursive_sum()
814 struct blkcg_gq *blkg; in blkg_conf_prep() local
841 blkg = blkg_lookup_check(blkcg, pol, q); in blkg_conf_prep()
842 if (IS_ERR(blkg)) { in blkg_conf_prep()
843 ret = PTR_ERR(blkg); in blkg_conf_prep()
847 if (blkg) in blkg_conf_prep()
878 blkg = blkg_lookup_check(pos, pol, q); in blkg_conf_prep()
879 if (IS_ERR(blkg)) { in blkg_conf_prep()
880 ret = PTR_ERR(blkg); in blkg_conf_prep()
884 if (blkg) { in blkg_conf_prep()
887 blkg = blkg_create(pos, q, new_blkg); in blkg_conf_prep()
888 if (unlikely(IS_ERR(blkg))) { in blkg_conf_prep()
889 ret = PTR_ERR(blkg); in blkg_conf_prep()
899 ctx->blkg = blkg; in blkg_conf_prep()
947 struct blkcg_gq *blkg; in blkcg_print_stat() local
951 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
956 dname = blkg_dev_name(blkg); in blkcg_print_stat()
960 spin_lock_irq(blkg->q->queue_lock); in blkcg_print_stat()
962 rwstat = blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
967 rwstat = blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
972 spin_unlock_irq(blkg->q->queue_lock); in blkcg_print_stat()
1018 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_css_offline() local
1020 struct request_queue *q = blkg->q; in blkcg_css_offline()
1023 blkg_destroy(blkg); in blkcg_css_offline()
1134 struct blkcg_gq *new_blkg, *blkg; in blkcg_init_queue() local
1151 blkg = blkg_create(&blkcg_root, q, new_blkg); in blkcg_init_queue()
1152 if (IS_ERR(blkg)) in blkcg_init_queue()
1154 q->root_blkg = blkg; in blkcg_init_queue()
1155 q->root_rl.blkg = blkg; in blkcg_init_queue()
1175 return PTR_ERR(blkg); in blkcg_init_queue()
1299 struct blkcg_gq *blkg; in blkcg_activate_policy() local
1320 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1323 if (blkg->pd[pol->plid]) in blkcg_activate_policy()
1334 blkg->pd[pol->plid] = pd; in blkcg_activate_policy()
1335 pd->blkg = blkg; in blkcg_activate_policy()
1367 struct blkcg_gq *blkg; in blkcg_deactivate_policy() local
1381 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_deactivate_policy()
1382 if (blkg->pd[pol->plid]) { in blkcg_deactivate_policy()
1384 pol->pd_offline_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1385 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1386 blkg->pd[pol->plid] = NULL; in blkcg_deactivate_policy()