Lines Matching refs:ssp
46 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
83 static void init_srcu_struct_nodes(struct srcu_struct *ssp, bool is_static) in init_srcu_struct_nodes() argument
97 ssp->level[0] = &ssp->node[0]; in init_srcu_struct_nodes()
99 ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1]; in init_srcu_struct_nodes()
103 srcu_for_each_node_breadth_first(ssp, snp) { in init_srcu_struct_nodes()
114 if (snp == &ssp->node[0]) { in init_srcu_struct_nodes()
121 if (snp == ssp->level[level + 1]) in init_srcu_struct_nodes()
123 snp->srcu_parent = ssp->level[level - 1] + in init_srcu_struct_nodes()
124 (snp - ssp->level[level]) / in init_srcu_struct_nodes()
135 snp_first = ssp->level[level]; in init_srcu_struct_nodes()
137 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_nodes()
141 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq; in init_srcu_struct_nodes()
142 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq; in init_srcu_struct_nodes()
152 sdp->ssp = ssp; in init_srcu_struct_nodes()
171 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) in init_srcu_struct_fields() argument
173 mutex_init(&ssp->srcu_cb_mutex); in init_srcu_struct_fields()
174 mutex_init(&ssp->srcu_gp_mutex); in init_srcu_struct_fields()
175 ssp->srcu_idx = 0; in init_srcu_struct_fields()
176 ssp->srcu_gp_seq = 0; in init_srcu_struct_fields()
177 ssp->srcu_barrier_seq = 0; in init_srcu_struct_fields()
178 mutex_init(&ssp->srcu_barrier_mutex); in init_srcu_struct_fields()
179 atomic_set(&ssp->srcu_barrier_cpu_cnt, 0); in init_srcu_struct_fields()
180 INIT_DELAYED_WORK(&ssp->work, process_srcu); in init_srcu_struct_fields()
182 ssp->sda = alloc_percpu(struct srcu_data); in init_srcu_struct_fields()
183 if (!ssp->sda) in init_srcu_struct_fields()
185 init_srcu_struct_nodes(ssp, is_static); in init_srcu_struct_fields()
186 ssp->srcu_gp_seq_needed_exp = 0; in init_srcu_struct_fields()
187 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); in init_srcu_struct_fields()
188 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */ in init_srcu_struct_fields()
194 int __init_srcu_struct(struct srcu_struct *ssp, const char *name, in __init_srcu_struct() argument
198 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); in __init_srcu_struct()
199 lockdep_init_map(&ssp->dep_map, name, key, 0); in __init_srcu_struct()
200 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in __init_srcu_struct()
201 return init_srcu_struct_fields(ssp, false); in __init_srcu_struct()
215 int init_srcu_struct(struct srcu_struct *ssp) in init_srcu_struct() argument
217 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in init_srcu_struct()
218 return init_srcu_struct_fields(ssp, false); in init_srcu_struct()
232 static void check_init_srcu_struct(struct srcu_struct *ssp) in check_init_srcu_struct() argument
237 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq_needed))) /*^^^*/ in check_init_srcu_struct()
239 spin_lock_irqsave_rcu_node(ssp, flags); in check_init_srcu_struct()
240 if (!rcu_seq_state(ssp->srcu_gp_seq_needed)) { in check_init_srcu_struct()
241 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
244 init_srcu_struct_fields(ssp, true); in check_init_srcu_struct()
245 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
252 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_lock_idx() argument
258 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_lock_idx()
269 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_unlock_idx() argument
275 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_unlock_idx()
286 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) in srcu_readers_active_idx_check() argument
290 unlocks = srcu_readers_unlock_idx(ssp, idx); in srcu_readers_active_idx_check()
326 return srcu_readers_lock_idx(ssp, idx) == unlocks; in srcu_readers_active_idx_check()
338 static bool srcu_readers_active(struct srcu_struct *ssp) in srcu_readers_active() argument
344 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_active()
360 static unsigned long srcu_get_delay(struct srcu_struct *ssp) in srcu_get_delay() argument
362 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), in srcu_get_delay()
363 READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_get_delay()
375 void cleanup_srcu_struct(struct srcu_struct *ssp) in cleanup_srcu_struct() argument
379 if (WARN_ON(!srcu_get_delay(ssp))) in cleanup_srcu_struct()
381 if (WARN_ON(srcu_readers_active(ssp))) in cleanup_srcu_struct()
383 flush_delayed_work(&ssp->work); in cleanup_srcu_struct()
385 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); in cleanup_srcu_struct()
392 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) || in cleanup_srcu_struct()
393 WARN_ON(srcu_readers_active(ssp))) { in cleanup_srcu_struct()
395 __func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq))); in cleanup_srcu_struct()
398 free_percpu(ssp->sda); in cleanup_srcu_struct()
399 ssp->sda = NULL; in cleanup_srcu_struct()
408 int __srcu_read_lock(struct srcu_struct *ssp) in __srcu_read_lock() argument
412 idx = READ_ONCE(ssp->srcu_idx) & 0x1; in __srcu_read_lock()
413 this_cpu_inc(ssp->sda->srcu_lock_count[idx]); in __srcu_read_lock()
424 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) in __srcu_read_unlock() argument
427 this_cpu_inc(ssp->sda->srcu_unlock_count[idx]); in __srcu_read_unlock()
443 static void srcu_gp_start(struct srcu_struct *ssp) in srcu_gp_start() argument
445 struct srcu_data *sdp = this_cpu_ptr(ssp->sda); in srcu_gp_start()
448 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); in srcu_gp_start()
449 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_gp_start()
452 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start()
454 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_gp_start()
457 rcu_seq_start(&ssp->srcu_gp_seq); in srcu_gp_start()
458 state = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_start()
496 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_schedule_cbs_snp() argument
504 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); in srcu_schedule_cbs_snp()
517 static void srcu_gp_end(struct srcu_struct *ssp) in srcu_gp_end() argument
531 mutex_lock(&ssp->srcu_cb_mutex); in srcu_gp_end()
534 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
535 idx = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_end()
537 cbdelay = srcu_get_delay(ssp); in srcu_gp_end()
538 WRITE_ONCE(ssp->srcu_last_gp_end, ktime_get_mono_fast_ns()); in srcu_gp_end()
539 rcu_seq_end(&ssp->srcu_gp_seq); in srcu_gp_end()
540 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
541 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq)) in srcu_gp_end()
542 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq); in srcu_gp_end()
543 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
544 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_gp_end()
549 srcu_for_each_node_breadth_first(ssp, snp) { in srcu_gp_end()
552 last_lvl = snp >= ssp->level[rcu_num_lvls - 1]; in srcu_gp_end()
563 srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay); in srcu_gp_end()
568 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_gp_end()
581 mutex_unlock(&ssp->srcu_cb_mutex); in srcu_gp_end()
584 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
585 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
587 ULONG_CMP_LT(gpseq, ssp->srcu_gp_seq_needed)) { in srcu_gp_end()
588 srcu_gp_start(ssp); in srcu_gp_end()
589 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
590 srcu_reschedule(ssp, 0); in srcu_gp_end()
592 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
603 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_funnel_exp_start() argument
609 if (rcu_seq_done(&ssp->srcu_gp_seq, s) || in srcu_funnel_exp_start()
620 spin_lock_irqsave_rcu_node(ssp, flags); in srcu_funnel_exp_start()
621 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_exp_start()
622 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_exp_start()
623 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_exp_start()
636 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, in srcu_funnel_gp_start() argument
646 if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode) in srcu_funnel_gp_start()
661 srcu_funnel_exp_start(ssp, snp, s); in srcu_funnel_gp_start()
673 spin_lock_irqsave_rcu_node(ssp, flags); in srcu_funnel_gp_start()
674 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) { in srcu_funnel_gp_start()
679 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/ in srcu_funnel_gp_start()
681 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_gp_start()
682 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_gp_start()
685 if (!rcu_seq_done(&ssp->srcu_gp_seq, s) && in srcu_funnel_gp_start()
686 rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) { in srcu_funnel_gp_start()
687 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_funnel_gp_start()
688 srcu_gp_start(ssp); in srcu_funnel_gp_start()
690 queue_delayed_work(rcu_gp_wq, &ssp->work, in srcu_funnel_gp_start()
691 srcu_get_delay(ssp)); in srcu_funnel_gp_start()
692 else if (list_empty(&ssp->work.work.entry)) in srcu_funnel_gp_start()
693 list_add(&ssp->work.work.entry, &srcu_boot_list); in srcu_funnel_gp_start()
695 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_gp_start()
703 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) in try_check_zero() argument
706 if (srcu_readers_active_idx_check(ssp, idx)) in try_check_zero()
708 if (--trycount + !srcu_get_delay(ssp) <= 0) in try_check_zero()
719 static void srcu_flip(struct srcu_struct *ssp) in srcu_flip() argument
731 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); in srcu_flip()
764 static bool srcu_might_be_idle(struct srcu_struct *ssp) in srcu_might_be_idle() argument
772 check_init_srcu_struct(ssp); in srcu_might_be_idle()
774 sdp = raw_cpu_ptr(ssp->sda); in srcu_might_be_idle()
790 tlast = READ_ONCE(ssp->srcu_last_gp_end); in srcu_might_be_idle()
796 curseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_might_be_idle()
798 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_gp_seq_needed))) in srcu_might_be_idle()
801 if (curseq != rcu_seq_current(&ssp->srcu_gp_seq)) in srcu_might_be_idle()
816 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, in srcu_gp_start_if_needed() argument
826 check_init_srcu_struct(ssp); in srcu_gp_start_if_needed()
827 idx = srcu_read_lock(ssp); in srcu_gp_start_if_needed()
828 sdp = raw_cpu_ptr(ssp->sda); in srcu_gp_start_if_needed()
833 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start_if_needed()
834 s = rcu_seq_snap(&ssp->srcu_gp_seq); in srcu_gp_start_if_needed()
846 srcu_funnel_gp_start(ssp, sdp, s, do_norm); in srcu_gp_start_if_needed()
848 srcu_funnel_exp_start(ssp, sdp->mynode, s); in srcu_gp_start_if_needed()
849 srcu_read_unlock(ssp, idx); in srcu_gp_start_if_needed()
881 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in __call_srcu() argument
891 (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); in __call_srcu()
911 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument
914 __call_srcu(ssp, rhp, func, true); in call_srcu()
921 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) in __synchronize_srcu() argument
925 RCU_LOCKDEP_WARN(lock_is_held(&ssp->dep_map) || in __synchronize_srcu()
934 check_init_srcu_struct(ssp); in __synchronize_srcu()
937 __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); in __synchronize_srcu()
961 void synchronize_srcu_expedited(struct srcu_struct *ssp) in synchronize_srcu_expedited() argument
963 __synchronize_srcu(ssp, rcu_gp_is_normal()); in synchronize_srcu_expedited()
1011 void synchronize_srcu(struct srcu_struct *ssp) in synchronize_srcu() argument
1013 if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited()) in synchronize_srcu()
1014 synchronize_srcu_expedited(ssp); in synchronize_srcu()
1016 __synchronize_srcu(ssp, true); in synchronize_srcu()
1030 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) in get_state_synchronize_srcu() argument
1035 return rcu_seq_snap(&ssp->srcu_gp_seq); in get_state_synchronize_srcu()
1049 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) in start_poll_synchronize_srcu() argument
1051 return srcu_gp_start_if_needed(ssp, NULL, true); in start_poll_synchronize_srcu()
1065 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) in poll_state_synchronize_srcu() argument
1067 if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie)) in poll_state_synchronize_srcu()
1082 struct srcu_struct *ssp; in srcu_barrier_cb() local
1085 ssp = sdp->ssp; in srcu_barrier_cb()
1086 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier_cb()
1087 complete(&ssp->srcu_barrier_completion); in srcu_barrier_cb()
1094 void srcu_barrier(struct srcu_struct *ssp) in srcu_barrier() argument
1098 unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq); in srcu_barrier()
1100 check_init_srcu_struct(ssp); in srcu_barrier()
1101 mutex_lock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1102 if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) { in srcu_barrier()
1104 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1107 rcu_seq_start(&ssp->srcu_barrier_seq); in srcu_barrier()
1108 init_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1111 atomic_set(&ssp->srcu_barrier_cpu_cnt, 1); in srcu_barrier()
1122 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_barrier()
1124 atomic_inc(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier()
1130 atomic_dec(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier()
1136 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier()
1137 complete(&ssp->srcu_barrier_completion); in srcu_barrier()
1138 wait_for_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1140 rcu_seq_end(&ssp->srcu_barrier_seq); in srcu_barrier()
1141 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1152 unsigned long srcu_batches_completed(struct srcu_struct *ssp) in srcu_batches_completed() argument
1154 return READ_ONCE(ssp->srcu_idx); in srcu_batches_completed()
1163 static void srcu_advance_state(struct srcu_struct *ssp) in srcu_advance_state() argument
1167 mutex_lock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1179 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */ in srcu_advance_state()
1181 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1182 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_advance_state()
1183 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq)); in srcu_advance_state()
1184 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1185 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1188 idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)); in srcu_advance_state()
1190 srcu_gp_start(ssp); in srcu_advance_state()
1191 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1193 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1198 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) { in srcu_advance_state()
1199 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1200 if (!try_check_zero(ssp, idx, 1)) { in srcu_advance_state()
1201 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1204 srcu_flip(ssp); in srcu_advance_state()
1205 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1206 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2); in srcu_advance_state()
1207 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1210 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) { in srcu_advance_state()
1216 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1217 if (!try_check_zero(ssp, idx, 2)) { in srcu_advance_state()
1218 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1221 srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ in srcu_advance_state()
1237 struct srcu_struct *ssp; in srcu_invoke_callbacks() local
1241 ssp = sdp->ssp; in srcu_invoke_callbacks()
1245 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1271 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1283 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) in srcu_reschedule() argument
1287 spin_lock_irq_rcu_node(ssp); in srcu_reschedule()
1288 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_reschedule()
1289 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) { in srcu_reschedule()
1293 } else if (!rcu_seq_state(ssp->srcu_gp_seq)) { in srcu_reschedule()
1295 srcu_gp_start(ssp); in srcu_reschedule()
1297 spin_unlock_irq_rcu_node(ssp); in srcu_reschedule()
1300 queue_delayed_work(rcu_gp_wq, &ssp->work, delay); in srcu_reschedule()
1308 struct srcu_struct *ssp; in process_srcu() local
1310 ssp = container_of(work, struct srcu_struct, work.work); in process_srcu()
1312 srcu_advance_state(ssp); in process_srcu()
1313 srcu_reschedule(ssp, srcu_get_delay(ssp)); in process_srcu()
1317 struct srcu_struct *ssp, int *flags, in srcutorture_get_gp_data() argument
1323 *gp_seq = rcu_seq_current(&ssp->srcu_gp_seq); in srcutorture_get_gp_data()
1327 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) in srcu_torture_stats_print() argument
1333 idx = ssp->srcu_idx & 0x1; in srcu_torture_stats_print()
1335 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), idx); in srcu_torture_stats_print()
1342 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_torture_stats_print()
1378 struct srcu_struct *ssp; in srcu_init() local
1382 ssp = list_first_entry(&srcu_boot_list, struct srcu_struct, in srcu_init()
1384 check_init_srcu_struct(ssp); in srcu_init()
1385 list_del_init(&ssp->work.work.entry); in srcu_init()
1386 queue_work(rcu_gp_wq, &ssp->work.work); in srcu_init()