1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright IBM Corporation, 2001
6 *
7 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
8 * Manfred Spraul <manfred@colorfullife.com>
9 *
10 * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
11 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
12 * Papers:
13 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
14 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
15 *
16 * For detailed explanation of Read-Copy Update mechanism see -
17 * http://lse.sourceforge.net/locking/rcupdate.html
18 *
19 */
20 #include <linux/types.h>
21 #include <linux/kernel.h>
22 #include <linux/init.h>
23 #include <linux/spinlock.h>
24 #include <linux/smp.h>
25 #include <linux/interrupt.h>
26 #include <linux/sched/signal.h>
27 #include <linux/sched/debug.h>
28 #include <linux/torture.h>
29 #include <linux/atomic.h>
30 #include <linux/bitops.h>
31 #include <linux/percpu.h>
32 #include <linux/notifier.h>
33 #include <linux/cpu.h>
34 #include <linux/mutex.h>
35 #include <linux/export.h>
36 #include <linux/hardirq.h>
37 #include <linux/delay.h>
38 #include <linux/moduleparam.h>
39 #include <linux/kthread.h>
40 #include <linux/tick.h>
41 #include <linux/rcupdate_wait.h>
42 #include <linux/sched/isolation.h>
43 #include <linux/kprobes.h>
44 #include <linux/slab.h>
45 #include <linux/irq_work.h>
46 #include <linux/rcupdate_trace.h>
47 #include <linux/jiffies.h>
48
49 #define CREATE_TRACE_POINTS
50
51 #include "rcu.h"
52
53 #ifdef MODULE_PARAM_PREFIX
54 #undef MODULE_PARAM_PREFIX
55 #endif
56 #define MODULE_PARAM_PREFIX "rcupdate."
57
58 #ifndef CONFIG_TINY_RCU
59 module_param(rcu_expedited, int, 0444);
60 module_param(rcu_normal, int, 0444);
61 static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT);
62 #if !defined(CONFIG_PREEMPT_RT) || defined(CONFIG_NO_HZ_FULL)
63 module_param(rcu_normal_after_boot, int, 0444);
64 #endif
65 #endif /* #ifndef CONFIG_TINY_RCU */
66
67 #ifdef CONFIG_DEBUG_LOCK_ALLOC
68 /**
69 * rcu_read_lock_held_common() - might we be in RCU-sched read-side critical section?
70 * @ret: Best guess answer if lockdep cannot be relied on
71 *
72 * Returns true if lockdep must be ignored, in which case ``*ret`` contains
73 * the best guess described below. Otherwise returns false, in which
74 * case ``*ret`` tells the caller nothing and the caller should instead
75 * consult lockdep.
76 *
77 * If CONFIG_DEBUG_LOCK_ALLOC is selected, set ``*ret`` to nonzero iff in an
78 * RCU-sched read-side critical section. In absence of
79 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
80 * critical section unless it can prove otherwise. Note that disabling
81 * of preemption (including disabling irqs) counts as an RCU-sched
82 * read-side critical section. This is useful for debug checks in functions
83 * that required that they be called within an RCU-sched read-side
84 * critical section.
85 *
86 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
87 * and while lockdep is disabled.
88 *
89 * Note that if the CPU is in the idle loop from an RCU point of view (ie:
90 * that we are in the section between ct_idle_enter() and ct_idle_exit())
91 * then rcu_read_lock_held() sets ``*ret`` to false even if the CPU did an
92 * rcu_read_lock(). The reason for this is that RCU ignores CPUs that are
93 * in such a section, considering these as in extended quiescent state,
94 * so such a CPU is effectively never in an RCU read-side critical section
95 * regardless of what RCU primitives it invokes. This state of affairs is
96 * required --- we need to keep an RCU-free window in idle where the CPU may
97 * possibly enter into low power mode. This way we can notice an extended
98 * quiescent state to other CPUs that started a grace period. Otherwise
99 * we would delay any grace period as long as we run in the idle task.
100 *
101 * Similarly, we avoid claiming an RCU read lock held if the current
102 * CPU is offline.
103 */
rcu_read_lock_held_common(bool * ret)104 static bool rcu_read_lock_held_common(bool *ret)
105 {
106 if (!debug_lockdep_rcu_enabled()) {
107 *ret = true;
108 return true;
109 }
110 if (!rcu_is_watching()) {
111 *ret = false;
112 return true;
113 }
114 if (!rcu_lockdep_current_cpu_online()) {
115 *ret = false;
116 return true;
117 }
118 return false;
119 }
120
rcu_read_lock_sched_held(void)121 int rcu_read_lock_sched_held(void)
122 {
123 bool ret;
124
125 if (rcu_read_lock_held_common(&ret))
126 return ret;
127 return lock_is_held(&rcu_sched_lock_map) || !preemptible();
128 }
129 EXPORT_SYMBOL(rcu_read_lock_sched_held);
130 #endif
131
132 #ifndef CONFIG_TINY_RCU
133
134 /*
135 * Should expedited grace-period primitives always fall back to their
136 * non-expedited counterparts? Intended for use within RCU. Note
137 * that if the user specifies both rcu_expedited and rcu_normal, then
138 * rcu_normal wins. (Except during the time period during boot from
139 * when the first task is spawned until the rcu_set_runtime_mode()
140 * core_initcall() is invoked, at which point everything is expedited.)
141 */
rcu_gp_is_normal(void)142 bool rcu_gp_is_normal(void)
143 {
144 return READ_ONCE(rcu_normal) &&
145 rcu_scheduler_active != RCU_SCHEDULER_INIT;
146 }
147 EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
148
149 static atomic_t rcu_async_hurry_nesting = ATOMIC_INIT(1);
150 /*
151 * Should call_rcu() callbacks be processed with urgency or are
152 * they OK being executed with arbitrary delays?
153 */
rcu_async_should_hurry(void)154 bool rcu_async_should_hurry(void)
155 {
156 return !IS_ENABLED(CONFIG_RCU_LAZY) ||
157 atomic_read(&rcu_async_hurry_nesting);
158 }
159 EXPORT_SYMBOL_GPL(rcu_async_should_hurry);
160
161 /**
162 * rcu_async_hurry - Make future async RCU callbacks not lazy.
163 *
164 * After a call to this function, future calls to call_rcu()
165 * will be processed in a timely fashion.
166 */
rcu_async_hurry(void)167 void rcu_async_hurry(void)
168 {
169 if (IS_ENABLED(CONFIG_RCU_LAZY))
170 atomic_inc(&rcu_async_hurry_nesting);
171 }
172 EXPORT_SYMBOL_GPL(rcu_async_hurry);
173
174 /**
175 * rcu_async_relax - Make future async RCU callbacks lazy.
176 *
177 * After a call to this function, future calls to call_rcu()
178 * will be processed in a lazy fashion.
179 */
rcu_async_relax(void)180 void rcu_async_relax(void)
181 {
182 if (IS_ENABLED(CONFIG_RCU_LAZY))
183 atomic_dec(&rcu_async_hurry_nesting);
184 }
185 EXPORT_SYMBOL_GPL(rcu_async_relax);
186
187 static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
188 /*
189 * Should normal grace-period primitives be expedited? Intended for
190 * use within RCU. Note that this function takes the rcu_expedited
191 * sysfs/boot variable and rcu_scheduler_active into account as well
192 * as the rcu_expedite_gp() nesting. So looping on rcu_unexpedite_gp()
193 * until rcu_gp_is_expedited() returns false is a -really- bad idea.
194 */
rcu_gp_is_expedited(void)195 bool rcu_gp_is_expedited(void)
196 {
197 return rcu_expedited || atomic_read(&rcu_expedited_nesting);
198 }
199 EXPORT_SYMBOL_GPL(rcu_gp_is_expedited);
200
201 /**
202 * rcu_expedite_gp - Expedite future RCU grace periods
203 *
204 * After a call to this function, future calls to synchronize_rcu() and
205 * friends act as the corresponding synchronize_rcu_expedited() function
206 * had instead been called.
207 */
rcu_expedite_gp(void)208 void rcu_expedite_gp(void)
209 {
210 atomic_inc(&rcu_expedited_nesting);
211 }
212 EXPORT_SYMBOL_GPL(rcu_expedite_gp);
213
214 /**
215 * rcu_unexpedite_gp - Cancel prior rcu_expedite_gp() invocation
216 *
217 * Undo a prior call to rcu_expedite_gp(). If all prior calls to
218 * rcu_expedite_gp() are undone by a subsequent call to rcu_unexpedite_gp(),
219 * and if the rcu_expedited sysfs/boot parameter is not set, then all
220 * subsequent calls to synchronize_rcu() and friends will return to
221 * their normal non-expedited behavior.
222 */
rcu_unexpedite_gp(void)223 void rcu_unexpedite_gp(void)
224 {
225 atomic_dec(&rcu_expedited_nesting);
226 }
227 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
228
229 /*
230 * Minimum time in milliseconds from the start boot until RCU can consider
231 * in-kernel boot as completed. This can also be tuned at runtime to end the
232 * boot earlier, by userspace init code writing the time in milliseconds (even
233 * 0) to: /sys/module/rcupdate/parameters/rcu_boot_end_delay. The sysfs node
234 * can also be used to extend the delay to be larger than the default, assuming
235 * the marking of boot complete has not yet occurred.
236 */
237 static int rcu_boot_end_delay = CONFIG_RCU_BOOT_END_DELAY;
238
239 static bool rcu_boot_ended __read_mostly;
240 static bool rcu_boot_end_called __read_mostly;
241 static DEFINE_MUTEX(rcu_boot_end_lock);
242
243 /*
244 * Inform RCU of the end of the in-kernel boot sequence. The boot sequence will
245 * not be marked ended until at least rcu_boot_end_delay milliseconds have passed.
246 */
247 void rcu_end_inkernel_boot(void);
rcu_boot_end_work_fn(struct work_struct * work)248 static void rcu_boot_end_work_fn(struct work_struct *work)
249 {
250 rcu_end_inkernel_boot();
251 }
252 static DECLARE_DELAYED_WORK(rcu_boot_end_work, rcu_boot_end_work_fn);
253
254 /* Must be called with rcu_boot_end_lock held. */
rcu_end_inkernel_boot_locked(void)255 static void rcu_end_inkernel_boot_locked(void)
256 {
257 rcu_boot_end_called = true;
258
259 if (rcu_boot_ended)
260 return;
261
262 if (rcu_boot_end_delay) {
263 u64 boot_ms = div_u64(ktime_get_boot_fast_ns(), 1000000UL);
264
265 if (boot_ms < rcu_boot_end_delay) {
266 schedule_delayed_work(&rcu_boot_end_work,
267 msecs_to_jiffies(rcu_boot_end_delay - boot_ms));
268 return;
269 }
270 }
271
272 cancel_delayed_work(&rcu_boot_end_work);
273 rcu_unexpedite_gp();
274 rcu_async_relax();
275 if (rcu_normal_after_boot)
276 WRITE_ONCE(rcu_normal, 1);
277 rcu_boot_ended = true;
278 }
279
rcu_end_inkernel_boot(void)280 void rcu_end_inkernel_boot(void)
281 {
282 mutex_lock(&rcu_boot_end_lock);
283 rcu_end_inkernel_boot_locked();
284 mutex_unlock(&rcu_boot_end_lock);
285 }
286
param_set_rcu_boot_end(const char * val,const struct kernel_param * kp)287 static int param_set_rcu_boot_end(const char *val, const struct kernel_param *kp)
288 {
289 uint end_ms;
290 int ret = kstrtouint(val, 0, &end_ms);
291
292 if (ret)
293 return ret;
294 /*
295 * rcu_end_inkernel_boot() should be called at least once during init
296 * before we can allow param changes to end the boot.
297 */
298 mutex_lock(&rcu_boot_end_lock);
299 rcu_boot_end_delay = end_ms;
300 if (!rcu_boot_ended && rcu_boot_end_called) {
301 rcu_end_inkernel_boot_locked();
302 }
303 mutex_unlock(&rcu_boot_end_lock);
304 return ret;
305 }
306
307 static const struct kernel_param_ops rcu_boot_end_ops = {
308 .set = param_set_rcu_boot_end,
309 .get = param_get_uint,
310 };
311 module_param_cb(rcu_boot_end_delay, &rcu_boot_end_ops, &rcu_boot_end_delay, 0644);
312
313 /*
314 * Let rcutorture know when it is OK to turn it up to eleven.
315 */
rcu_inkernel_boot_has_ended(void)316 bool rcu_inkernel_boot_has_ended(void)
317 {
318 return rcu_boot_ended;
319 }
320 EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
321
322 #endif /* #ifndef CONFIG_TINY_RCU */
323
324 /*
325 * Test each non-SRCU synchronous grace-period wait API. This is
326 * useful just after a change in mode for these primitives, and
327 * during early boot.
328 */
rcu_test_sync_prims(void)329 void rcu_test_sync_prims(void)
330 {
331 if (!IS_ENABLED(CONFIG_PROVE_RCU))
332 return;
333 pr_info("Running RCU synchronous self tests\n");
334 synchronize_rcu();
335 synchronize_rcu_expedited();
336 }
337
338 #if !defined(CONFIG_TINY_RCU)
339
340 /*
341 * Switch to run-time mode once RCU has fully initialized.
342 */
rcu_set_runtime_mode(void)343 static int __init rcu_set_runtime_mode(void)
344 {
345 rcu_test_sync_prims();
346 rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
347 kfree_rcu_scheduler_running();
348 rcu_test_sync_prims();
349 return 0;
350 }
351 core_initcall(rcu_set_runtime_mode);
352
353 #endif /* #if !defined(CONFIG_TINY_RCU) */
354
355 #ifdef CONFIG_DEBUG_LOCK_ALLOC
356 static struct lock_class_key rcu_lock_key;
357 struct lockdep_map rcu_lock_map = {
358 .name = "rcu_read_lock",
359 .key = &rcu_lock_key,
360 .wait_type_outer = LD_WAIT_FREE,
361 .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_RT implies PREEMPT_RCU */
362 };
363 EXPORT_SYMBOL_GPL(rcu_lock_map);
364
365 static struct lock_class_key rcu_bh_lock_key;
366 struct lockdep_map rcu_bh_lock_map = {
367 .name = "rcu_read_lock_bh",
368 .key = &rcu_bh_lock_key,
369 .wait_type_outer = LD_WAIT_FREE,
370 .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_RT makes BH preemptible. */
371 };
372 EXPORT_SYMBOL_GPL(rcu_bh_lock_map);
373
374 static struct lock_class_key rcu_sched_lock_key;
375 struct lockdep_map rcu_sched_lock_map = {
376 .name = "rcu_read_lock_sched",
377 .key = &rcu_sched_lock_key,
378 .wait_type_outer = LD_WAIT_FREE,
379 .wait_type_inner = LD_WAIT_SPIN,
380 };
381 EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
382
383 // Tell lockdep when RCU callbacks are being invoked.
384 static struct lock_class_key rcu_callback_key;
385 struct lockdep_map rcu_callback_map =
386 STATIC_LOCKDEP_MAP_INIT("rcu_callback", &rcu_callback_key);
387 EXPORT_SYMBOL_GPL(rcu_callback_map);
388
debug_lockdep_rcu_enabled(void)389 noinstr int notrace debug_lockdep_rcu_enabled(void)
390 {
391 return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && READ_ONCE(debug_locks) &&
392 current->lockdep_recursion == 0;
393 }
394 EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
395
396 /**
397 * rcu_read_lock_held() - might we be in RCU read-side critical section?
398 *
399 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
400 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
401 * this assumes we are in an RCU read-side critical section unless it can
402 * prove otherwise. This is useful for debug checks in functions that
403 * require that they be called within an RCU read-side critical section.
404 *
405 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
406 * and while lockdep is disabled.
407 *
408 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
409 * occur in the same context, for example, it is illegal to invoke
410 * rcu_read_unlock() in process context if the matching rcu_read_lock()
411 * was invoked from within an irq handler.
412 *
413 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
414 * offline from an RCU perspective, so check for those as well.
415 */
rcu_read_lock_held(void)416 int rcu_read_lock_held(void)
417 {
418 bool ret;
419
420 if (rcu_read_lock_held_common(&ret))
421 return ret;
422 return lock_is_held(&rcu_lock_map);
423 }
424 EXPORT_SYMBOL_GPL(rcu_read_lock_held);
425
426 /**
427 * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
428 *
429 * Check for bottom half being disabled, which covers both the
430 * CONFIG_PROVE_RCU and not cases. Note that if someone uses
431 * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
432 * will show the situation. This is useful for debug checks in functions
433 * that require that they be called within an RCU read-side critical
434 * section.
435 *
436 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
437 *
438 * Note that rcu_read_lock_bh() is disallowed if the CPU is either idle or
439 * offline from an RCU perspective, so check for those as well.
440 */
rcu_read_lock_bh_held(void)441 int rcu_read_lock_bh_held(void)
442 {
443 bool ret;
444
445 if (rcu_read_lock_held_common(&ret))
446 return ret;
447 return in_softirq() || irqs_disabled();
448 }
449 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
450
rcu_read_lock_any_held(void)451 int rcu_read_lock_any_held(void)
452 {
453 bool ret;
454
455 if (rcu_read_lock_held_common(&ret))
456 return ret;
457 if (lock_is_held(&rcu_lock_map) ||
458 lock_is_held(&rcu_bh_lock_map) ||
459 lock_is_held(&rcu_sched_lock_map))
460 return 1;
461 return !preemptible();
462 }
463 EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
464
465 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
466
467 /**
468 * wakeme_after_rcu() - Callback function to awaken a task after grace period
469 * @head: Pointer to rcu_head member within rcu_synchronize structure
470 *
471 * Awaken the corresponding task now that a grace period has elapsed.
472 */
wakeme_after_rcu(struct rcu_head * head)473 void wakeme_after_rcu(struct rcu_head *head)
474 {
475 struct rcu_synchronize *rcu;
476
477 rcu = container_of(head, struct rcu_synchronize, head);
478 complete(&rcu->completion);
479 }
480 EXPORT_SYMBOL_GPL(wakeme_after_rcu);
481
__wait_rcu_gp(bool checktiny,unsigned int state,int n,call_rcu_func_t * crcu_array,struct rcu_synchronize * rs_array)482 void __wait_rcu_gp(bool checktiny, unsigned int state, int n, call_rcu_func_t *crcu_array,
483 struct rcu_synchronize *rs_array)
484 {
485 int i;
486 int j;
487
488 /* Initialize and register callbacks for each crcu_array element. */
489 for (i = 0; i < n; i++) {
490 if (checktiny &&
491 (crcu_array[i] == call_rcu)) {
492 might_sleep();
493 continue;
494 }
495 for (j = 0; j < i; j++)
496 if (crcu_array[j] == crcu_array[i])
497 break;
498 if (j == i) {
499 init_rcu_head_on_stack(&rs_array[i].head);
500 init_completion(&rs_array[i].completion);
501 (crcu_array[i])(&rs_array[i].head, wakeme_after_rcu);
502 }
503 }
504
505 /* Wait for all callbacks to be invoked. */
506 for (i = 0; i < n; i++) {
507 if (checktiny &&
508 (crcu_array[i] == call_rcu))
509 continue;
510 for (j = 0; j < i; j++)
511 if (crcu_array[j] == crcu_array[i])
512 break;
513 if (j == i) {
514 wait_for_completion_state(&rs_array[i].completion, state);
515 destroy_rcu_head_on_stack(&rs_array[i].head);
516 }
517 }
518 }
519 EXPORT_SYMBOL_GPL(__wait_rcu_gp);
520
finish_rcuwait(struct rcuwait * w)521 void finish_rcuwait(struct rcuwait *w)
522 {
523 rcu_assign_pointer(w->task, NULL);
524 __set_current_state(TASK_RUNNING);
525 }
526 EXPORT_SYMBOL_GPL(finish_rcuwait);
527
528 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
init_rcu_head(struct rcu_head * head)529 void init_rcu_head(struct rcu_head *head)
530 {
531 debug_object_init(head, &rcuhead_debug_descr);
532 }
533 EXPORT_SYMBOL_GPL(init_rcu_head);
534
destroy_rcu_head(struct rcu_head * head)535 void destroy_rcu_head(struct rcu_head *head)
536 {
537 debug_object_free(head, &rcuhead_debug_descr);
538 }
539 EXPORT_SYMBOL_GPL(destroy_rcu_head);
540
rcuhead_is_static_object(void * addr)541 static bool rcuhead_is_static_object(void *addr)
542 {
543 return true;
544 }
545
546 /**
547 * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
548 * @head: pointer to rcu_head structure to be initialized
549 *
550 * This function informs debugobjects of a new rcu_head structure that
551 * has been allocated as an auto variable on the stack. This function
552 * is not required for rcu_head structures that are statically defined or
553 * that are dynamically allocated on the heap. This function has no
554 * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
555 */
init_rcu_head_on_stack(struct rcu_head * head)556 void init_rcu_head_on_stack(struct rcu_head *head)
557 {
558 debug_object_init_on_stack(head, &rcuhead_debug_descr);
559 }
560 EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
561
562 /**
563 * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
564 * @head: pointer to rcu_head structure to be initialized
565 *
566 * This function informs debugobjects that an on-stack rcu_head structure
567 * is about to go out of scope. As with init_rcu_head_on_stack(), this
568 * function is not required for rcu_head structures that are statically
569 * defined or that are dynamically allocated on the heap. Also as with
570 * init_rcu_head_on_stack(), this function has no effect for
571 * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
572 */
destroy_rcu_head_on_stack(struct rcu_head * head)573 void destroy_rcu_head_on_stack(struct rcu_head *head)
574 {
575 debug_object_free(head, &rcuhead_debug_descr);
576 }
577 EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
578
579 const struct debug_obj_descr rcuhead_debug_descr = {
580 .name = "rcu_head",
581 .is_static_object = rcuhead_is_static_object,
582 };
583 EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
584 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
585
586 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_RCU_TRACE)
do_trace_rcu_torture_read(const char * rcutorturename,struct rcu_head * rhp,unsigned long secs,unsigned long c_old,unsigned long c)587 void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
588 unsigned long secs,
589 unsigned long c_old, unsigned long c)
590 {
591 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c);
592 }
593 EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
594 #else
595 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
596 do { } while (0)
597 #endif
598
599 #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST)
600 /* Get rcutorture access to sched_setaffinity(). */
torture_sched_setaffinity(pid_t pid,const struct cpumask * in_mask)601 long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
602 {
603 int ret;
604
605 ret = sched_setaffinity(pid, in_mask);
606 WARN_ONCE(ret, "%s: sched_setaffinity(%d) returned %d\n", __func__, pid, ret);
607 return ret;
608 }
609 EXPORT_SYMBOL_GPL(torture_sched_setaffinity);
610 #endif
611
612 int rcu_cpu_stall_notifiers __read_mostly; // !0 = provide stall notifiers (rarely useful)
613 EXPORT_SYMBOL_GPL(rcu_cpu_stall_notifiers);
614
615 #ifdef CONFIG_RCU_STALL_COMMON
616 int rcu_cpu_stall_ftrace_dump __read_mostly;
617 module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
618 #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
619 module_param(rcu_cpu_stall_notifiers, int, 0444);
620 #endif // #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
621 int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
622 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
623 module_param(rcu_cpu_stall_suppress, int, 0644);
624 int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
625 module_param(rcu_cpu_stall_timeout, int, 0644);
626 int rcu_exp_cpu_stall_timeout __read_mostly = CONFIG_RCU_EXP_CPU_STALL_TIMEOUT;
627 module_param(rcu_exp_cpu_stall_timeout, int, 0644);
628 int rcu_cpu_stall_cputime __read_mostly = IS_ENABLED(CONFIG_RCU_CPU_STALL_CPUTIME);
629 module_param(rcu_cpu_stall_cputime, int, 0644);
630 bool rcu_exp_stall_task_details __read_mostly;
631 module_param(rcu_exp_stall_task_details, bool, 0644);
632 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
633
634 // Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
635 // warnings. Also used by rcutorture even if stall warnings are excluded.
636 int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
637 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
638 module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
639
640 /**
641 * get_completed_synchronize_rcu - Return a pre-completed polled state cookie
642 *
643 * Returns a value that will always be treated by functions like
644 * poll_state_synchronize_rcu() as a cookie whose grace period has already
645 * completed.
646 */
get_completed_synchronize_rcu(void)647 unsigned long get_completed_synchronize_rcu(void)
648 {
649 return RCU_GET_STATE_COMPLETED;
650 }
651 EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu);
652
653 #ifdef CONFIG_PROVE_RCU
654
655 /*
656 * Early boot self test parameters.
657 */
658 static bool rcu_self_test;
659 module_param(rcu_self_test, bool, 0444);
660
661 static int rcu_self_test_counter;
662
test_callback(struct rcu_head * r)663 static void test_callback(struct rcu_head *r)
664 {
665 rcu_self_test_counter++;
666 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
667 }
668
669 DEFINE_STATIC_SRCU(early_srcu);
670 static unsigned long early_srcu_cookie;
671
672 struct early_boot_kfree_rcu {
673 struct rcu_head rh;
674 };
675
early_boot_test_call_rcu(void)676 static void early_boot_test_call_rcu(void)
677 {
678 static struct rcu_head head;
679 int idx;
680 static struct rcu_head shead;
681 struct early_boot_kfree_rcu *rhp;
682
683 idx = srcu_down_read(&early_srcu);
684 srcu_up_read(&early_srcu, idx);
685 call_rcu(&head, test_callback);
686 early_srcu_cookie = start_poll_synchronize_srcu(&early_srcu);
687 call_srcu(&early_srcu, &shead, test_callback);
688 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
689 if (!WARN_ON_ONCE(!rhp))
690 kfree_rcu(rhp, rh);
691 }
692
rcu_early_boot_tests(void)693 void rcu_early_boot_tests(void)
694 {
695 pr_info("Running RCU self tests\n");
696
697 if (rcu_self_test)
698 early_boot_test_call_rcu();
699 rcu_test_sync_prims();
700 }
701
rcu_verify_early_boot_tests(void)702 static int rcu_verify_early_boot_tests(void)
703 {
704 int ret = 0;
705 int early_boot_test_counter = 0;
706
707 if (rcu_self_test) {
708 early_boot_test_counter++;
709 rcu_barrier();
710 early_boot_test_counter++;
711 srcu_barrier(&early_srcu);
712 WARN_ON_ONCE(!poll_state_synchronize_srcu(&early_srcu, early_srcu_cookie));
713 cleanup_srcu_struct(&early_srcu);
714 }
715 if (rcu_self_test_counter != early_boot_test_counter) {
716 WARN_ON(1);
717 ret = -1;
718 }
719
720 return ret;
721 }
722 late_initcall(rcu_verify_early_boot_tests);
723 #else
rcu_early_boot_tests(void)724 void rcu_early_boot_tests(void) {}
725 #endif /* CONFIG_PROVE_RCU */
726
727 #include "tasks.h"
728
729 #ifndef CONFIG_TINY_RCU
730
731 /*
732 * Print any significant non-default boot-time settings.
733 */
rcupdate_announce_bootup_oddness(void)734 void __init rcupdate_announce_bootup_oddness(void)
735 {
736 if (rcu_normal)
737 pr_info("\tNo expedited grace period (rcu_normal).\n");
738 else if (rcu_normal_after_boot)
739 pr_info("\tNo expedited grace period (rcu_normal_after_boot).\n");
740 else if (rcu_expedited)
741 pr_info("\tAll grace periods are expedited (rcu_expedited).\n");
742 if (rcu_cpu_stall_suppress)
743 pr_info("\tRCU CPU stall warnings suppressed (rcu_cpu_stall_suppress).\n");
744 if (rcu_cpu_stall_timeout != CONFIG_RCU_CPU_STALL_TIMEOUT)
745 pr_info("\tRCU CPU stall warnings timeout set to %d (rcu_cpu_stall_timeout).\n", rcu_cpu_stall_timeout);
746 rcu_tasks_bootup_oddness();
747 }
748
749 #endif /* #ifndef CONFIG_TINY_RCU */
750