72 	jea->key	= jeb->key - delta;  in jump_label_swap()
76 	jeb->key	= tmp.key + delta;  in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key)  in static_key_count()  argument
110 	int n = atomic_read(&key->enabled);  in static_key_count()
116 void static_key_slow_inc_cpuslocked(struct static_key *key)  in static_key_slow_inc_cpuslocked()  argument
120 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_inc_cpuslocked()
135 	for (v = atomic_read(&key->enabled); v > 0; v = v1) {  in static_key_slow_inc_cpuslocked()
136 		v1 = atomic_cmpxchg(&key->enabled, v, v + 1);  in static_key_slow_inc_cpuslocked()
142 	if (atomic_read(&key->enabled) == 0) {  in static_key_slow_inc_cpuslocked()
143 		atomic_set(&key->enabled, -1);  in static_key_slow_inc_cpuslocked()
144 		jump_label_update(key);  in static_key_slow_inc_cpuslocked()
149 		atomic_set_release(&key->enabled, 1);  in static_key_slow_inc_cpuslocked()
151 		atomic_inc(&key->enabled);  in static_key_slow_inc_cpuslocked()
156 void static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
159 	static_key_slow_inc_cpuslocked(key);  in static_key_slow_inc()
164 void static_key_enable_cpuslocked(struct static_key *key)  in static_key_enable_cpuslocked()  argument
166 	STATIC_KEY_CHECK_USE(key);  in static_key_enable_cpuslocked()
169 	if (atomic_read(&key->enabled) > 0) {  in static_key_enable_cpuslocked()
170 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable_cpuslocked()
175 	if (atomic_read(&key->enabled) == 0) {  in static_key_enable_cpuslocked()
176 		atomic_set(&key->enabled, -1);  in static_key_enable_cpuslocked()
177 		jump_label_update(key);  in static_key_enable_cpuslocked()
181 		atomic_set_release(&key->enabled, 1);  in static_key_enable_cpuslocked()
187 void static_key_enable(struct static_key *key)  in static_key_enable()  argument
190 	static_key_enable_cpuslocked(key);  in static_key_enable()
195 void static_key_disable_cpuslocked(struct static_key *key)  in static_key_disable_cpuslocked()  argument
197 	STATIC_KEY_CHECK_USE(key);  in static_key_disable_cpuslocked()
200 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable_cpuslocked()
201 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable_cpuslocked()
206 	if (atomic_cmpxchg(&key->enabled, 1, 0))  in static_key_disable_cpuslocked()
207 		jump_label_update(key);  in static_key_disable_cpuslocked()
212 void static_key_disable(struct static_key *key)  in static_key_disable()  argument
215 	static_key_disable_cpuslocked(key);  in static_key_disable()
220 static bool static_key_slow_try_dec(struct static_key *key)  in static_key_slow_try_dec()  argument
224 	val = atomic_fetch_add_unless(&key->enabled, -1, 1);  in static_key_slow_try_dec()
239 static void __static_key_slow_dec_cpuslocked(struct static_key *key)  in __static_key_slow_dec_cpuslocked()  argument
243 	if (static_key_slow_try_dec(key))  in __static_key_slow_dec_cpuslocked()
247 	if (atomic_dec_and_test(&key->enabled))  in __static_key_slow_dec_cpuslocked()
248 		jump_label_update(key);  in __static_key_slow_dec_cpuslocked()
252 static void __static_key_slow_dec(struct static_key *key)  in __static_key_slow_dec()  argument
255 	__static_key_slow_dec_cpuslocked(key);  in __static_key_slow_dec()
261 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
263 	__static_key_slow_dec(&key->key);  in jump_label_update_timeout()
267 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
269 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
270 	__static_key_slow_dec(key);  in static_key_slow_dec()
274 void static_key_slow_dec_cpuslocked(struct static_key *key)  in static_key_slow_dec_cpuslocked()  argument
276 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_cpuslocked()
277 	__static_key_slow_dec_cpuslocked(key);  in static_key_slow_dec_cpuslocked()
280 void __static_key_slow_dec_deferred(struct static_key *key,  in __static_key_slow_dec_deferred()  argument
284 	STATIC_KEY_CHECK_USE(key);  in __static_key_slow_dec_deferred()
286 	if (static_key_slow_try_dec(key))  in __static_key_slow_dec_deferred()
293 void __static_key_deferred_flush(void *key, struct delayed_work *work)  in __static_key_deferred_flush()  argument
295 	STATIC_KEY_CHECK_USE(key);  in __static_key_deferred_flush()
300 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
303 	STATIC_KEY_CHECK_USE(key);  in jump_label_rate_limit()
304 	key->timeout = rl;  in jump_label_rate_limit()
305 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
347 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
349 	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);  in static_key_entries()
350 	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);  in static_key_entries()
353 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
355 	return key->type & JUMP_TYPE_TRUE;  in static_key_type()
358 static inline bool static_key_linked(struct static_key *key)  in static_key_linked()  argument
360 	return key->type & JUMP_TYPE_LINKED;  in static_key_linked()
363 static inline void static_key_clear_linked(struct static_key *key)  in static_key_clear_linked()  argument
365 	key->type &= ~JUMP_TYPE_LINKED;  in static_key_clear_linked()
368 static inline void static_key_set_linked(struct static_key *key)  in static_key_set_linked()  argument
370 	key->type |= JUMP_TYPE_LINKED;  in static_key_set_linked()
382 static void static_key_set_entries(struct static_key *key,  in static_key_set_entries()  argument
388 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_entries()
389 	key->entries = entries;  in static_key_set_entries()
390 	key->type |= type;  in static_key_set_entries()
395 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
396 	bool enabled = static_key_enabled(key);  in jump_label_type()
430 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
435 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
441 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
446 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
467 	struct static_key *key = NULL;  in jump_label_init()  local
497 		if (iterk == key)  in jump_label_init()
500 		key = iterk;  in jump_label_init()
501 		static_key_set_entries(key, iter);  in jump_label_init()
512 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
513 	bool type = static_key_type(key);  in jump_label_init_type()
526 static inline struct static_key_mod *static_key_mod(struct static_key *key)  in static_key_mod()  argument
528 	WARN_ON_ONCE(!static_key_linked(key));  in static_key_mod()
529 	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);  in static_key_mod()
538 static void static_key_set_mod(struct static_key *key,  in static_key_set_mod()  argument
544 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_mod()
545 	key->next = mod;  in static_key_set_mod()
546 	key->type |= type;  in static_key_set_mod()
573 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
577 	for (mod = static_key_mod(key); mod; mod = mod->next) {  in __jump_label_mod_update()
593 		__jump_label_update(key, mod->entries, stop,  in __jump_label_mod_update()
628 	struct static_key *key = NULL;  in jump_label_add_module()  local
644 		if (iterk == key)  in jump_label_add_module()
647 		key = iterk;  in jump_label_add_module()
648 		if (within_module((unsigned long)key, mod)) {  in jump_label_add_module()
649 			static_key_set_entries(key, iter);  in jump_label_add_module()
655 		if (!static_key_linked(key)) {  in jump_label_add_module()
663 			jlm2->mod = __module_address((unsigned long)key);  in jump_label_add_module()
665 			jlm2->entries = static_key_entries(key);  in jump_label_add_module()
667 			static_key_set_mod(key, jlm2);  in jump_label_add_module()
668 			static_key_set_linked(key);  in jump_label_add_module()
672 		jlm->next = static_key_mod(key);  in jump_label_add_module()
673 		static_key_set_mod(key, jlm);  in jump_label_add_module()
674 		static_key_set_linked(key);  in jump_label_add_module()
678 			__jump_label_update(key, iter, iter_stop, true);  in jump_label_add_module()
689 	struct static_key *key = NULL;  in jump_label_del_module()  local
693 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
696 		key = jump_entry_key(iter);  in jump_label_del_module()
698 		if (within_module((unsigned long)key, mod))  in jump_label_del_module()
702 		if (WARN_ON(!static_key_linked(key)))  in jump_label_del_module()
705 		prev = &key->next;  in jump_label_del_module()
706 		jlm = static_key_mod(key);  in jump_label_del_module()
717 		if (prev == &key->next)  in jump_label_del_module()
718 			static_key_set_mod(key, jlm->next);  in jump_label_del_module()
724 		jlm = static_key_mod(key);  in jump_label_del_module()
727 			static_key_set_entries(key, jlm->entries);  in jump_label_del_module()
728 			static_key_clear_linked(key);  in jump_label_del_module()
804 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
811 	if (static_key_linked(key)) {  in jump_label_update()
812 		__jump_label_mod_update(key);  in jump_label_update()
817 	mod = __module_address((unsigned long)key);  in jump_label_update()
822 	entry = static_key_entries(key);  in jump_label_update()
825 		__jump_label_update(key, entry, stop,  in jump_label_update()
838 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
839 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
849 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
850 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()