84 	pd->pg[0].shift = shift;  in prop_descriptor_init()
86 	err = percpu_counter_init(&pd->pg[0].events, 0, gfp);  in prop_descriptor_init()
90 	err = percpu_counter_init(&pd->pg[1].events, 0, gfp);  in prop_descriptor_init()
92 		percpu_counter_destroy(&pd->pg[0].events);  in prop_descriptor_init()
118 	offset = pd->pg[pd->index].shift - shift;  in prop_change_shift()
122 	pd->pg[index].shift = shift;  in prop_change_shift()
125 	events = percpu_counter_sum(&pd->pg[pd->index].events);  in prop_change_shift()
130 	percpu_counter_set(&pd->pg[index].events, events);  in prop_change_shift()
160 	return &pd->pg[index];  in prop_get_global()
163 static void prop_put_global(struct prop_descriptor *pd, struct prop_global *pg)  in prop_put_global()  argument
212 void prop_norm_percpu(struct prop_global *pg, struct prop_local_percpu *pl)  in prop_norm_percpu()  argument
214 	unsigned long period = 1UL << (pg->shift - 1);  in prop_norm_percpu()
219 	global_period = percpu_counter_read(&pg->events);  in prop_norm_percpu()
230 	prop_adjust_shift(&pl->shift, &pl->period, pg->shift);  in prop_norm_percpu()
237 	period = (global_period - pl->period) >> (pg->shift - 1);  in prop_norm_percpu()
258 	struct prop_global *pg = prop_get_global(pd);  in __prop_inc_percpu()  local
260 	prop_norm_percpu(pg, pl);  in __prop_inc_percpu()
262 	percpu_counter_add(&pg->events, 1);  in __prop_inc_percpu()
263 	prop_put_global(pd, pg);  in __prop_inc_percpu()
273 	struct prop_global *pg = prop_get_global(pd);  in __prop_inc_percpu_max()  local
275 	prop_norm_percpu(pg, pl);  in __prop_inc_percpu_max()
278 		unsigned long period_2 = 1UL << (pg->shift - 1);  in __prop_inc_percpu_max()
284 		global_count = percpu_counter_read(&pg->events);  in __prop_inc_percpu_max()
292 	percpu_counter_add(&pg->events, 1);  in __prop_inc_percpu_max()
295 	prop_put_global(pd, pg);  in __prop_inc_percpu_max()
307 	struct prop_global *pg = prop_get_global(pd);  in prop_fraction_percpu()  local
308 	unsigned long period_2 = 1UL << (pg->shift - 1);  in prop_fraction_percpu()
312 	prop_norm_percpu(pg, pl);  in prop_fraction_percpu()
315 	global_count = percpu_counter_read(&pg->events);  in prop_fraction_percpu()
318 	prop_put_global(pd, pg);  in prop_fraction_percpu()
342 void prop_norm_single(struct prop_global *pg, struct prop_local_single *pl)  in prop_norm_single()  argument
344 	unsigned long period = 1UL << (pg->shift - 1);  in prop_norm_single()
349 	global_period = percpu_counter_read(&pg->events);  in prop_norm_single()
360 	prop_adjust_shift(&pl->shift, &pl->period, pg->shift);  in prop_norm_single()
364 	period = (global_period - pl->period) >> (pg->shift - 1);  in prop_norm_single()
378 	struct prop_global *pg = prop_get_global(pd);  in __prop_inc_single()  local
380 	prop_norm_single(pg, pl);  in __prop_inc_single()
382 	percpu_counter_add(&pg->events, 1);  in __prop_inc_single()
383 	prop_put_global(pd, pg);  in __prop_inc_single()
395 	struct prop_global *pg = prop_get_global(pd);  in prop_fraction_single()  local
396 	unsigned long period_2 = 1UL << (pg->shift - 1);  in prop_fraction_single()
400 	prop_norm_single(pg, pl);  in prop_fraction_single()
403 	global_count = percpu_counter_read(&pg->events);  in prop_fraction_single()
406 	prop_put_global(pd, pg);  in prop_fraction_single()