24 	struct percpu_counter *fbc = addr;  in percpu_counter_fixup_free()  local
28 		percpu_counter_destroy(fbc);  in percpu_counter_fixup_free()
29 		debug_object_free(fbc, &percpu_counter_debug_descr);  in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
43 	debug_object_init(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
44 	debug_object_activate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
49 	debug_object_deactivate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
50 	debug_object_free(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)  in percpu_counter_set()  argument
65 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_set()
67 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_set()
70 	fbc->count = amount;  in percpu_counter_set()
71 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_set()
82 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)  in percpu_counter_add_batch()  argument
87 	count = __this_cpu_read(*fbc->counters) + amount;  in percpu_counter_add_batch()
90 		raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_add_batch()
91 		fbc->count += count;  in percpu_counter_add_batch()
92 		__this_cpu_sub(*fbc->counters, count - amount);  in percpu_counter_add_batch()
93 		raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_add_batch()
95 		this_cpu_add(*fbc->counters, amount);  in percpu_counter_add_batch()
105 s64 __percpu_counter_sum(struct percpu_counter *fbc)  in __percpu_counter_sum()  argument
111 	raw_spin_lock_irqsave(&fbc->lock, flags);  in __percpu_counter_sum()
112 	ret = fbc->count;  in __percpu_counter_sum()
114 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in __percpu_counter_sum()
117 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in __percpu_counter_sum()
122 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,  in __percpu_counter_init()  argument
127 	raw_spin_lock_init(&fbc->lock);  in __percpu_counter_init()
128 	lockdep_set_class(&fbc->lock, key);  in __percpu_counter_init()
129 	fbc->count = amount;  in __percpu_counter_init()
130 	fbc->counters = alloc_percpu_gfp(s32, gfp);  in __percpu_counter_init()
131 	if (!fbc->counters)  in __percpu_counter_init()
134 	debug_percpu_counter_activate(fbc);  in __percpu_counter_init()
137 	INIT_LIST_HEAD(&fbc->list);  in __percpu_counter_init()
139 	list_add(&fbc->list, &percpu_counters);  in __percpu_counter_init()
146 void percpu_counter_destroy(struct percpu_counter *fbc)  in percpu_counter_destroy()  argument
150 	if (!fbc->counters)  in percpu_counter_destroy()
153 	debug_percpu_counter_deactivate(fbc);  in percpu_counter_destroy()
157 	list_del(&fbc->list);  in percpu_counter_destroy()
160 	free_percpu(fbc->counters);  in percpu_counter_destroy()
161 	fbc->counters = NULL;  in percpu_counter_destroy()
179 	struct percpu_counter *fbc;  in percpu_counter_cpu_dead()  local
184 	list_for_each_entry(fbc, &percpu_counters, list) {  in percpu_counter_cpu_dead()
187 		raw_spin_lock(&fbc->lock);  in percpu_counter_cpu_dead()
188 		pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_cpu_dead()
189 		fbc->count += *pcount;  in percpu_counter_cpu_dead()
191 		raw_spin_unlock(&fbc->lock);  in percpu_counter_cpu_dead()
202 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)  in __percpu_counter_compare()  argument
206 	count = percpu_counter_read(fbc);  in __percpu_counter_compare()
215 	count = percpu_counter_sum(fbc);  in __percpu_counter_compare()