13 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
15 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
17 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
29 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
31 	raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);  in cgroup_rstat_updated()
42 	if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next))  in cgroup_rstat_updated()
49 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
66 		prstatc = cgroup_rstat_cpu(parent, cpu);  in cgroup_rstat_updated()
92 						   struct cgroup *root, int cpu)  in cgroup_rstat_cpu_pop_updated()  argument
107 		if (!cgroup_rstat_cpu(pos, cpu)->updated_next)  in cgroup_rstat_cpu_pop_updated()
115 		rstatc = cgroup_rstat_cpu(pos, cpu);  in cgroup_rstat_cpu_pop_updated()
132 		prstatc = cgroup_rstat_cpu(parent, cpu);  in cgroup_rstat_cpu_pop_updated()
137 			nrstatc = cgroup_rstat_cpu(*nextp, cpu);  in cgroup_rstat_cpu_pop_updated()
167 				     struct cgroup *parent, int cpu)  in bpf_rstat_flush()  argument
177 	int cpu;  in cgroup_rstat_flush_locked()  local
181 	for_each_possible_cpu(cpu) {  in cgroup_rstat_flush_locked()
183 						       cpu);  in cgroup_rstat_flush_locked()
196 		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {  in cgroup_rstat_flush_locked()
199 			cgroup_base_stat_flush(pos, cpu);  in cgroup_rstat_flush_locked()
200 			bpf_rstat_flush(pos, cgroup_parent(pos), cpu);  in cgroup_rstat_flush_locked()
205 				css->ss->css_rstat_flush(css, cpu);  in cgroup_rstat_flush_locked()
286 	int cpu;  in cgroup_rstat_init()  local
296 	for_each_possible_cpu(cpu) {  in cgroup_rstat_init()
297 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
308 	int cpu;  in cgroup_rstat_exit()  local
313 	for_each_possible_cpu(cpu) {  in cgroup_rstat_exit()
314 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
327 	int cpu;  in cgroup_rstat_boot()  local
329 	for_each_possible_cpu(cpu)  in cgroup_rstat_boot()
330 		raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu));  in cgroup_rstat_boot()
359 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
361 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()