36 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,  in __hugetlb_cgroup_counter_from_cgroup()  argument
40 		return &h_cg->rsvd_hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
41 	return &h_cg->hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
45 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup()  argument
47 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);  in hugetlb_cgroup_counter_from_cgroup()
51 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup_rsvd()  argument
53 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);  in hugetlb_cgroup_counter_from_cgroup_rsvd()
68 static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_is_root()  argument
70 	return (h_cg == root_h_cgroup);  in hugetlb_cgroup_is_root()
74 parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)  in parent_hugetlb_cgroup()  argument
76 	return hugetlb_cgroup_from_css(h_cg->css.parent);  in parent_hugetlb_cgroup()
79 static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_have_usage()  argument
85 				hugetlb_cgroup_counter_from_cgroup(h_cg, idx)))  in hugetlb_cgroup_have_usage()
161 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_move_parent()  argument
167 	struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);  in hugetlb_cgroup_move_parent()
175 	if (!page_hcg || page_hcg != h_cg)  in hugetlb_cgroup_move_parent()
184 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_move_parent()
199 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);  in hugetlb_cgroup_css_offline()  local
209 				hugetlb_cgroup_move_parent(idx, h_cg, page);  in hugetlb_cgroup_css_offline()
215 	} while (hugetlb_cgroup_have_usage(h_cg));  in hugetlb_cgroup_css_offline()
237 	struct hugetlb_cgroup *h_cg = NULL;  in __hugetlb_cgroup_charge_cgroup()  local
249 	h_cg = hugetlb_cgroup_from_task(current);  in __hugetlb_cgroup_charge_cgroup()
250 	if (!css_tryget(&h_cg->css)) {  in __hugetlb_cgroup_charge_cgroup()
257 		    __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in __hugetlb_cgroup_charge_cgroup()
260 		hugetlb_event(h_cg, idx, HUGETLB_MAX);  in __hugetlb_cgroup_charge_cgroup()
261 		css_put(&h_cg->css);  in __hugetlb_cgroup_charge_cgroup()
268 		css_put(&h_cg->css);  in __hugetlb_cgroup_charge_cgroup()
270 	*ptr = h_cg;  in __hugetlb_cgroup_charge_cgroup()
288 					   struct hugetlb_cgroup *h_cg,  in __hugetlb_cgroup_commit_charge()  argument
291 	if (hugetlb_cgroup_disabled() || !h_cg)  in __hugetlb_cgroup_commit_charge()
294 	__set_hugetlb_cgroup(page, h_cg, rsvd);  in __hugetlb_cgroup_commit_charge()
299 				  struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_commit_charge()  argument
302 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, false);  in hugetlb_cgroup_commit_charge()
306 				       struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_commit_charge_rsvd()  argument
309 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, true);  in hugetlb_cgroup_commit_charge_rsvd()
318 	struct hugetlb_cgroup *h_cg;  in __hugetlb_cgroup_uncharge_page()  local
323 	h_cg = __hugetlb_cgroup_from_page(page, rsvd);  in __hugetlb_cgroup_uncharge_page()
324 	if (unlikely(!h_cg))  in __hugetlb_cgroup_uncharge_page()
328 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_page()
333 		css_put(&h_cg->css);  in __hugetlb_cgroup_uncharge_page()
351 					     struct hugetlb_cgroup *h_cg,  in __hugetlb_cgroup_uncharge_cgroup()  argument
354 	if (hugetlb_cgroup_disabled() || !h_cg)  in __hugetlb_cgroup_uncharge_cgroup()
360 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_cgroup()
365 		css_put(&h_cg->css);  in __hugetlb_cgroup_uncharge_cgroup()
369 				    struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_uncharge_cgroup()  argument
371 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);  in hugetlb_cgroup_uncharge_cgroup()
375 					 struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_uncharge_cgroup_rsvd()  argument
377 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);  in hugetlb_cgroup_uncharge_cgroup_rsvd()
429 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);  in hugetlb_cgroup_read_u64()  local
431 	counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];  in hugetlb_cgroup_read_u64()
432 	rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)];  in hugetlb_cgroup_read_u64()
463 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));  in hugetlb_cgroup_read_u64_max()  local
466 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_read_u64_max()
473 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
480 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
504 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));  in hugetlb_cgroup_write()  local
507 	if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */  in hugetlb_cgroup_write()
525 			__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in hugetlb_cgroup_write()
553 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));  in hugetlb_cgroup_reset()  local
555 	counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];  in hugetlb_cgroup_reset()
556 	rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)];  in hugetlb_cgroup_reset()
594 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));  in __hugetlb_events_show()  local
599 		max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
601 		max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
780 	struct hugetlb_cgroup *h_cg;  in hugetlb_cgroup_migrate()  local
788 	h_cg = hugetlb_cgroup_from_page(oldhpage);  in hugetlb_cgroup_migrate()
794 	set_hugetlb_cgroup(newhpage, h_cg);  in hugetlb_cgroup_migrate()