• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #include <linux/kdebug.h>
2 #include <linux/kprobes.h>
3 #include <linux/export.h>
4 #include <linux/notifier.h>
5 #include <linux/rcupdate.h>
6 #include <linux/vmalloc.h>
7 #include <linux/reboot.h>
8 
9 /*
10  *	Notifier list for kernel code which wants to be called
11  *	at shutdown. This is used to stop any idling DMA operations
12  *	and the like.
13  */
14 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
15 
16 /*
17  *	Notifier chain core routines.  The exported routines below
18  *	are layered on top of these, with appropriate locking added.
19  */
20 
notifier_chain_register(struct notifier_block ** nl,struct notifier_block * n)21 static int notifier_chain_register(struct notifier_block **nl,
22 		struct notifier_block *n)
23 {
24 	while ((*nl) != NULL) {
25 		if (n->priority > (*nl)->priority)
26 			break;
27 		nl = &((*nl)->next);
28 	}
29 	n->next = *nl;
30 	rcu_assign_pointer(*nl, n);
31 	return 0;
32 }
33 
notifier_chain_cond_register(struct notifier_block ** nl,struct notifier_block * n)34 static int notifier_chain_cond_register(struct notifier_block **nl,
35 		struct notifier_block *n)
36 {
37 	while ((*nl) != NULL) {
38 		if ((*nl) == n)
39 			return 0;
40 		if (n->priority > (*nl)->priority)
41 			break;
42 		nl = &((*nl)->next);
43 	}
44 	n->next = *nl;
45 	rcu_assign_pointer(*nl, n);
46 	return 0;
47 }
48 
notifier_chain_unregister(struct notifier_block ** nl,struct notifier_block * n)49 static int notifier_chain_unregister(struct notifier_block **nl,
50 		struct notifier_block *n)
51 {
52 	while ((*nl) != NULL) {
53 		if ((*nl) == n) {
54 			rcu_assign_pointer(*nl, n->next);
55 			return 0;
56 		}
57 		nl = &((*nl)->next);
58 	}
59 	return -ENOENT;
60 }
61 
62 /**
63  * notifier_call_chain - Informs the registered notifiers about an event.
64  *	@nl:		Pointer to head of the blocking notifier chain
65  *	@val:		Value passed unmodified to notifier function
66  *	@v:		Pointer passed unmodified to notifier function
67  *	@nr_to_call:	Number of notifier functions to be called. Don't care
68  *			value of this parameter is -1.
69  *	@nr_calls:	Records the number of notifications sent. Don't care
70  *			value of this field is NULL.
71  *	@returns:	notifier_call_chain returns the value returned by the
72  *			last notifier function called.
73  */
notifier_call_chain(struct notifier_block ** nl,unsigned long val,void * v,int nr_to_call,int * nr_calls)74 static int notifier_call_chain(struct notifier_block **nl,
75 			       unsigned long val, void *v,
76 			       int nr_to_call, int *nr_calls)
77 {
78 	int ret = NOTIFY_DONE;
79 	struct notifier_block *nb, *next_nb;
80 
81 	nb = rcu_dereference_raw(*nl);
82 
83 	while (nb && nr_to_call) {
84 		next_nb = rcu_dereference_raw(nb->next);
85 
86 #ifdef CONFIG_DEBUG_NOTIFIERS
87 		if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
88 			WARN(1, "Invalid notifier called!");
89 			nb = next_nb;
90 			continue;
91 		}
92 #endif
93 		ret = nb->notifier_call(nb, val, v);
94 
95 		if (nr_calls)
96 			(*nr_calls)++;
97 
98 		if ((ret & NOTIFY_STOP_MASK) == NOTIFY_STOP_MASK)
99 			break;
100 		nb = next_nb;
101 		nr_to_call--;
102 	}
103 	return ret;
104 }
105 NOKPROBE_SYMBOL(notifier_call_chain);
106 
107 /*
108  *	Atomic notifier chain routines.  Registration and unregistration
109  *	use a spinlock, and call_chain is synchronized by RCU (no locks).
110  */
111 
112 /**
113  *	atomic_notifier_chain_register - Add notifier to an atomic notifier chain
114  *	@nh: Pointer to head of the atomic notifier chain
115  *	@n: New entry in notifier chain
116  *
117  *	Adds a notifier to an atomic notifier chain.
118  *
119  *	Currently always returns zero.
120  */
atomic_notifier_chain_register(struct atomic_notifier_head * nh,struct notifier_block * n)121 int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
122 		struct notifier_block *n)
123 {
124 	unsigned long flags;
125 	int ret;
126 
127 	spin_lock_irqsave(&nh->lock, flags);
128 	ret = notifier_chain_register(&nh->head, n);
129 	spin_unlock_irqrestore(&nh->lock, flags);
130 	return ret;
131 }
132 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
133 
134 /**
135  *	atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
136  *	@nh: Pointer to head of the atomic notifier chain
137  *	@n: Entry to remove from notifier chain
138  *
139  *	Removes a notifier from an atomic notifier chain.
140  *
141  *	Returns zero on success or %-ENOENT on failure.
142  */
atomic_notifier_chain_unregister(struct atomic_notifier_head * nh,struct notifier_block * n)143 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
144 		struct notifier_block *n)
145 {
146 	unsigned long flags;
147 	int ret;
148 
149 	spin_lock_irqsave(&nh->lock, flags);
150 	ret = notifier_chain_unregister(&nh->head, n);
151 	spin_unlock_irqrestore(&nh->lock, flags);
152 	synchronize_rcu();
153 	return ret;
154 }
155 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
156 
157 /**
158  *	__atomic_notifier_call_chain - Call functions in an atomic notifier chain
159  *	@nh: Pointer to head of the atomic notifier chain
160  *	@val: Value passed unmodified to notifier function
161  *	@v: Pointer passed unmodified to notifier function
162  *	@nr_to_call: See the comment for notifier_call_chain.
163  *	@nr_calls: See the comment for notifier_call_chain.
164  *
165  *	Calls each function in a notifier chain in turn.  The functions
166  *	run in an atomic context, so they must not block.
167  *	This routine uses RCU to synchronize with changes to the chain.
168  *
169  *	If the return value of the notifier can be and'ed
170  *	with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
171  *	will return immediately, with the return value of
172  *	the notifier function which halted execution.
173  *	Otherwise the return value is the return value
174  *	of the last notifier function called.
175  */
__atomic_notifier_call_chain(struct atomic_notifier_head * nh,unsigned long val,void * v,int nr_to_call,int * nr_calls)176 int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
177 				 unsigned long val, void *v,
178 				 int nr_to_call, int *nr_calls)
179 {
180 	int ret;
181 
182 	rcu_read_lock();
183 	ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
184 	rcu_read_unlock();
185 	return ret;
186 }
187 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
188 NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
189 
atomic_notifier_call_chain(struct atomic_notifier_head * nh,unsigned long val,void * v)190 int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
191 			       unsigned long val, void *v)
192 {
193 	return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
194 }
195 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
196 NOKPROBE_SYMBOL(atomic_notifier_call_chain);
197 
198 /*
199  *	Blocking notifier chain routines.  All access to the chain is
200  *	synchronized by an rwsem.
201  */
202 
203 /**
204  *	blocking_notifier_chain_register - Add notifier to a blocking notifier chain
205  *	@nh: Pointer to head of the blocking notifier chain
206  *	@n: New entry in notifier chain
207  *
208  *	Adds a notifier to a blocking notifier chain.
209  *	Must be called in process context.
210  *
211  *	Currently always returns zero.
212  */
blocking_notifier_chain_register(struct blocking_notifier_head * nh,struct notifier_block * n)213 int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
214 		struct notifier_block *n)
215 {
216 	int ret;
217 
218 	/*
219 	 * This code gets used during boot-up, when task switching is
220 	 * not yet working and interrupts must remain disabled.  At
221 	 * such times we must not call down_write().
222 	 */
223 	if (unlikely(system_state == SYSTEM_BOOTING))
224 		return notifier_chain_register(&nh->head, n);
225 
226 	down_write(&nh->rwsem);
227 	ret = notifier_chain_register(&nh->head, n);
228 	up_write(&nh->rwsem);
229 	return ret;
230 }
231 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
232 
233 /**
234  *	blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
235  *	@nh: Pointer to head of the blocking notifier chain
236  *	@n: New entry in notifier chain
237  *
238  *	Adds a notifier to a blocking notifier chain, only if not already
239  *	present in the chain.
240  *	Must be called in process context.
241  *
242  *	Currently always returns zero.
243  */
blocking_notifier_chain_cond_register(struct blocking_notifier_head * nh,struct notifier_block * n)244 int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh,
245 		struct notifier_block *n)
246 {
247 	int ret;
248 
249 	down_write(&nh->rwsem);
250 	ret = notifier_chain_cond_register(&nh->head, n);
251 	up_write(&nh->rwsem);
252 	return ret;
253 }
254 EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register);
255 
256 /**
257  *	blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
258  *	@nh: Pointer to head of the blocking notifier chain
259  *	@n: Entry to remove from notifier chain
260  *
261  *	Removes a notifier from a blocking notifier chain.
262  *	Must be called from process context.
263  *
264  *	Returns zero on success or %-ENOENT on failure.
265  */
blocking_notifier_chain_unregister(struct blocking_notifier_head * nh,struct notifier_block * n)266 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
267 		struct notifier_block *n)
268 {
269 	int ret;
270 
271 	/*
272 	 * This code gets used during boot-up, when task switching is
273 	 * not yet working and interrupts must remain disabled.  At
274 	 * such times we must not call down_write().
275 	 */
276 	if (unlikely(system_state == SYSTEM_BOOTING))
277 		return notifier_chain_unregister(&nh->head, n);
278 
279 	down_write(&nh->rwsem);
280 	ret = notifier_chain_unregister(&nh->head, n);
281 	up_write(&nh->rwsem);
282 	return ret;
283 }
284 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
285 
286 /**
287  *	__blocking_notifier_call_chain - Call functions in a blocking notifier chain
288  *	@nh: Pointer to head of the blocking notifier chain
289  *	@val: Value passed unmodified to notifier function
290  *	@v: Pointer passed unmodified to notifier function
291  *	@nr_to_call: See comment for notifier_call_chain.
292  *	@nr_calls: See comment for notifier_call_chain.
293  *
294  *	Calls each function in a notifier chain in turn.  The functions
295  *	run in a process context, so they are allowed to block.
296  *
297  *	If the return value of the notifier can be and'ed
298  *	with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
299  *	will return immediately, with the return value of
300  *	the notifier function which halted execution.
301  *	Otherwise the return value is the return value
302  *	of the last notifier function called.
303  */
__blocking_notifier_call_chain(struct blocking_notifier_head * nh,unsigned long val,void * v,int nr_to_call,int * nr_calls)304 int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
305 				   unsigned long val, void *v,
306 				   int nr_to_call, int *nr_calls)
307 {
308 	int ret = NOTIFY_DONE;
309 
310 	/*
311 	 * We check the head outside the lock, but if this access is
312 	 * racy then it does not matter what the result of the test
313 	 * is, we re-check the list after having taken the lock anyway:
314 	 */
315 	if (rcu_access_pointer(nh->head)) {
316 		down_read(&nh->rwsem);
317 		ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
318 					nr_calls);
319 		up_read(&nh->rwsem);
320 	}
321 	return ret;
322 }
323 EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
324 
blocking_notifier_call_chain(struct blocking_notifier_head * nh,unsigned long val,void * v)325 int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
326 		unsigned long val, void *v)
327 {
328 	return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
329 }
330 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
331 
332 /*
333  *	Raw notifier chain routines.  There is no protection;
334  *	the caller must provide it.  Use at your own risk!
335  */
336 
337 /**
338  *	raw_notifier_chain_register - Add notifier to a raw notifier chain
339  *	@nh: Pointer to head of the raw notifier chain
340  *	@n: New entry in notifier chain
341  *
342  *	Adds a notifier to a raw notifier chain.
343  *	All locking must be provided by the caller.
344  *
345  *	Currently always returns zero.
346  */
raw_notifier_chain_register(struct raw_notifier_head * nh,struct notifier_block * n)347 int raw_notifier_chain_register(struct raw_notifier_head *nh,
348 		struct notifier_block *n)
349 {
350 	return notifier_chain_register(&nh->head, n);
351 }
352 EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
353 
354 /**
355  *	raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
356  *	@nh: Pointer to head of the raw notifier chain
357  *	@n: Entry to remove from notifier chain
358  *
359  *	Removes a notifier from a raw notifier chain.
360  *	All locking must be provided by the caller.
361  *
362  *	Returns zero on success or %-ENOENT on failure.
363  */
raw_notifier_chain_unregister(struct raw_notifier_head * nh,struct notifier_block * n)364 int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
365 		struct notifier_block *n)
366 {
367 	return notifier_chain_unregister(&nh->head, n);
368 }
369 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
370 
371 /**
372  *	__raw_notifier_call_chain - Call functions in a raw notifier chain
373  *	@nh: Pointer to head of the raw notifier chain
374  *	@val: Value passed unmodified to notifier function
375  *	@v: Pointer passed unmodified to notifier function
376  *	@nr_to_call: See comment for notifier_call_chain.
377  *	@nr_calls: See comment for notifier_call_chain
378  *
379  *	Calls each function in a notifier chain in turn.  The functions
380  *	run in an undefined context.
381  *	All locking must be provided by the caller.
382  *
383  *	If the return value of the notifier can be and'ed
384  *	with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
385  *	will return immediately, with the return value of
386  *	the notifier function which halted execution.
387  *	Otherwise the return value is the return value
388  *	of the last notifier function called.
389  */
__raw_notifier_call_chain(struct raw_notifier_head * nh,unsigned long val,void * v,int nr_to_call,int * nr_calls)390 int __raw_notifier_call_chain(struct raw_notifier_head *nh,
391 			      unsigned long val, void *v,
392 			      int nr_to_call, int *nr_calls)
393 {
394 	return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
395 }
396 EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
397 
raw_notifier_call_chain(struct raw_notifier_head * nh,unsigned long val,void * v)398 int raw_notifier_call_chain(struct raw_notifier_head *nh,
399 		unsigned long val, void *v)
400 {
401 	return __raw_notifier_call_chain(nh, val, v, -1, NULL);
402 }
403 EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
404 
405 #ifdef CONFIG_SRCU
406 /*
407  *	SRCU notifier chain routines.    Registration and unregistration
408  *	use a mutex, and call_chain is synchronized by SRCU (no locks).
409  */
410 
411 /**
412  *	srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
413  *	@nh: Pointer to head of the SRCU notifier chain
414  *	@n: New entry in notifier chain
415  *
416  *	Adds a notifier to an SRCU notifier chain.
417  *	Must be called in process context.
418  *
419  *	Currently always returns zero.
420  */
srcu_notifier_chain_register(struct srcu_notifier_head * nh,struct notifier_block * n)421 int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
422 		struct notifier_block *n)
423 {
424 	int ret;
425 
426 	/*
427 	 * This code gets used during boot-up, when task switching is
428 	 * not yet working and interrupts must remain disabled.  At
429 	 * such times we must not call mutex_lock().
430 	 */
431 	if (unlikely(system_state == SYSTEM_BOOTING))
432 		return notifier_chain_register(&nh->head, n);
433 
434 	mutex_lock(&nh->mutex);
435 	ret = notifier_chain_register(&nh->head, n);
436 	mutex_unlock(&nh->mutex);
437 	return ret;
438 }
439 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
440 
441 /**
442  *	srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
443  *	@nh: Pointer to head of the SRCU notifier chain
444  *	@n: Entry to remove from notifier chain
445  *
446  *	Removes a notifier from an SRCU notifier chain.
447  *	Must be called from process context.
448  *
449  *	Returns zero on success or %-ENOENT on failure.
450  */
srcu_notifier_chain_unregister(struct srcu_notifier_head * nh,struct notifier_block * n)451 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
452 		struct notifier_block *n)
453 {
454 	int ret;
455 
456 	/*
457 	 * This code gets used during boot-up, when task switching is
458 	 * not yet working and interrupts must remain disabled.  At
459 	 * such times we must not call mutex_lock().
460 	 */
461 	if (unlikely(system_state == SYSTEM_BOOTING))
462 		return notifier_chain_unregister(&nh->head, n);
463 
464 	mutex_lock(&nh->mutex);
465 	ret = notifier_chain_unregister(&nh->head, n);
466 	mutex_unlock(&nh->mutex);
467 	synchronize_srcu(&nh->srcu);
468 	return ret;
469 }
470 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
471 
472 /**
473  *	__srcu_notifier_call_chain - Call functions in an SRCU notifier chain
474  *	@nh: Pointer to head of the SRCU notifier chain
475  *	@val: Value passed unmodified to notifier function
476  *	@v: Pointer passed unmodified to notifier function
477  *	@nr_to_call: See comment for notifier_call_chain.
478  *	@nr_calls: See comment for notifier_call_chain
479  *
480  *	Calls each function in a notifier chain in turn.  The functions
481  *	run in a process context, so they are allowed to block.
482  *
483  *	If the return value of the notifier can be and'ed
484  *	with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
485  *	will return immediately, with the return value of
486  *	the notifier function which halted execution.
487  *	Otherwise the return value is the return value
488  *	of the last notifier function called.
489  */
__srcu_notifier_call_chain(struct srcu_notifier_head * nh,unsigned long val,void * v,int nr_to_call,int * nr_calls)490 int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
491 			       unsigned long val, void *v,
492 			       int nr_to_call, int *nr_calls)
493 {
494 	int ret;
495 	int idx;
496 
497 	idx = srcu_read_lock(&nh->srcu);
498 	ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
499 	srcu_read_unlock(&nh->srcu, idx);
500 	return ret;
501 }
502 EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
503 
srcu_notifier_call_chain(struct srcu_notifier_head * nh,unsigned long val,void * v)504 int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
505 		unsigned long val, void *v)
506 {
507 	return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
508 }
509 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
510 
511 /**
512  *	srcu_init_notifier_head - Initialize an SRCU notifier head
513  *	@nh: Pointer to head of the srcu notifier chain
514  *
515  *	Unlike other sorts of notifier heads, SRCU notifier heads require
516  *	dynamic initialization.  Be sure to call this routine before
517  *	calling any of the other SRCU notifier routines for this head.
518  *
519  *	If an SRCU notifier head is deallocated, it must first be cleaned
520  *	up by calling srcu_cleanup_notifier_head().  Otherwise the head's
521  *	per-cpu data (used by the SRCU mechanism) will leak.
522  */
srcu_init_notifier_head(struct srcu_notifier_head * nh)523 void srcu_init_notifier_head(struct srcu_notifier_head *nh)
524 {
525 	mutex_init(&nh->mutex);
526 	if (init_srcu_struct(&nh->srcu) < 0)
527 		BUG();
528 	nh->head = NULL;
529 }
530 EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
531 
532 #endif /* CONFIG_SRCU */
533 
534 static ATOMIC_NOTIFIER_HEAD(die_chain);
535 
notify_die(enum die_val val,const char * str,struct pt_regs * regs,long err,int trap,int sig)536 int notrace notify_die(enum die_val val, const char *str,
537 	       struct pt_regs *regs, long err, int trap, int sig)
538 {
539 	struct die_args args = {
540 		.regs	= regs,
541 		.str	= str,
542 		.err	= err,
543 		.trapnr	= trap,
544 		.signr	= sig,
545 
546 	};
547 	RCU_LOCKDEP_WARN(!rcu_is_watching(),
548 			   "notify_die called but RCU thinks we're quiescent");
549 	return atomic_notifier_call_chain(&die_chain, val, &args);
550 }
551 NOKPROBE_SYMBOL(notify_die);
552 
register_die_notifier(struct notifier_block * nb)553 int register_die_notifier(struct notifier_block *nb)
554 {
555 	vmalloc_sync_mappings();
556 	return atomic_notifier_chain_register(&die_chain, nb);
557 }
558 EXPORT_SYMBOL_GPL(register_die_notifier);
559 
unregister_die_notifier(struct notifier_block * nb)560 int unregister_die_notifier(struct notifier_block *nb)
561 {
562 	return atomic_notifier_chain_unregister(&die_chain, nb);
563 }
564 EXPORT_SYMBOL_GPL(unregister_die_notifier);
565