Lines Matching refs:rsclp
91 void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) in rcu_segcblist_set_len() argument
94 atomic_long_set(&rsclp->len, v); in rcu_segcblist_set_len()
96 WRITE_ONCE(rsclp->len, v); in rcu_segcblist_set_len()
107 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v) in rcu_segcblist_add_len() argument
111 atomic_long_add(v, &rsclp->len); in rcu_segcblist_add_len()
115 WRITE_ONCE(rsclp->len, rsclp->len + v); in rcu_segcblist_add_len()
126 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp) in rcu_segcblist_inc_len() argument
128 rcu_segcblist_add_len(rsclp, 1); in rcu_segcblist_inc_len()
137 long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v) in rcu_segcblist_xchg_len() argument
140 return atomic_long_xchg(&rsclp->len, v); in rcu_segcblist_xchg_len()
142 long ret = rsclp->len; in rcu_segcblist_xchg_len()
145 WRITE_ONCE(rsclp->len, v); in rcu_segcblist_xchg_len()
154 void rcu_segcblist_init(struct rcu_segcblist *rsclp) in rcu_segcblist_init() argument
158 BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq)); in rcu_segcblist_init()
159 BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq)); in rcu_segcblist_init()
160 rsclp->head = NULL; in rcu_segcblist_init()
162 rsclp->tails[i] = &rsclp->head; in rcu_segcblist_init()
163 rcu_segcblist_set_len(rsclp, 0); in rcu_segcblist_init()
164 rsclp->len_lazy = 0; in rcu_segcblist_init()
165 rsclp->enabled = 1; in rcu_segcblist_init()
172 void rcu_segcblist_disable(struct rcu_segcblist *rsclp) in rcu_segcblist_disable() argument
174 WARN_ON_ONCE(!rcu_segcblist_empty(rsclp)); in rcu_segcblist_disable()
175 WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp)); in rcu_segcblist_disable()
176 WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp)); in rcu_segcblist_disable()
177 rsclp->enabled = 0; in rcu_segcblist_disable()
184 void rcu_segcblist_offload(struct rcu_segcblist *rsclp) in rcu_segcblist_offload() argument
186 rsclp->offloaded = 1; in rcu_segcblist_offload()
193 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp) in rcu_segcblist_ready_cbs() argument
195 return rcu_segcblist_is_enabled(rsclp) && in rcu_segcblist_ready_cbs()
196 &rsclp->head != rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_ready_cbs()
203 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp) in rcu_segcblist_pend_cbs() argument
205 return rcu_segcblist_is_enabled(rsclp) && in rcu_segcblist_pend_cbs()
206 !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL); in rcu_segcblist_pend_cbs()
213 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp) in rcu_segcblist_first_cb() argument
215 if (rcu_segcblist_is_enabled(rsclp)) in rcu_segcblist_first_cb()
216 return rsclp->head; in rcu_segcblist_first_cb()
227 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) in rcu_segcblist_first_pend_cb() argument
229 if (rcu_segcblist_is_enabled(rsclp)) in rcu_segcblist_first_pend_cb()
230 return *rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_first_pend_cb()
238 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) in rcu_segcblist_nextgp() argument
240 if (!rcu_segcblist_pend_cbs(rsclp)) in rcu_segcblist_nextgp()
242 *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; in rcu_segcblist_nextgp()
255 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, in rcu_segcblist_enqueue() argument
258 rcu_segcblist_inc_len(rsclp); in rcu_segcblist_enqueue()
260 rsclp->len_lazy++; in rcu_segcblist_enqueue()
263 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp); in rcu_segcblist_enqueue()
264 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next); in rcu_segcblist_enqueue()
277 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, in rcu_segcblist_entrain() argument
282 if (rcu_segcblist_n_cbs(rsclp) == 0) in rcu_segcblist_entrain()
284 rcu_segcblist_inc_len(rsclp); in rcu_segcblist_entrain()
286 rsclp->len_lazy++; in rcu_segcblist_entrain()
290 if (rsclp->tails[i] != rsclp->tails[i - 1]) in rcu_segcblist_entrain()
292 WRITE_ONCE(*rsclp->tails[i], rhp); in rcu_segcblist_entrain()
294 WRITE_ONCE(rsclp->tails[i], &rhp->next); in rcu_segcblist_entrain()
307 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, in rcu_segcblist_extract_count() argument
310 rclp->len_lazy += rsclp->len_lazy; in rcu_segcblist_extract_count()
311 rsclp->len_lazy = 0; in rcu_segcblist_extract_count()
312 rclp->len = rcu_segcblist_xchg_len(rsclp, 0); in rcu_segcblist_extract_count()
320 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, in rcu_segcblist_extract_done_cbs() argument
325 if (!rcu_segcblist_ready_cbs(rsclp)) in rcu_segcblist_extract_done_cbs()
327 *rclp->tail = rsclp->head; in rcu_segcblist_extract_done_cbs()
328 WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_extract_done_cbs()
329 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); in rcu_segcblist_extract_done_cbs()
330 rclp->tail = rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_done_cbs()
332 if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL]) in rcu_segcblist_extract_done_cbs()
333 WRITE_ONCE(rsclp->tails[i], &rsclp->head); in rcu_segcblist_extract_done_cbs()
343 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, in rcu_segcblist_extract_pend_cbs() argument
348 if (!rcu_segcblist_pend_cbs(rsclp)) in rcu_segcblist_extract_pend_cbs()
350 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_pend_cbs()
351 rclp->tail = rsclp->tails[RCU_NEXT_TAIL]; in rcu_segcblist_extract_pend_cbs()
352 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); in rcu_segcblist_extract_pend_cbs()
354 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_extract_pend_cbs()
361 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, in rcu_segcblist_insert_count() argument
364 rsclp->len_lazy += rclp->len_lazy; in rcu_segcblist_insert_count()
365 rcu_segcblist_add_len(rsclp, rclp->len); in rcu_segcblist_insert_count()
374 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, in rcu_segcblist_insert_done_cbs() argument
381 *rclp->tail = rsclp->head; in rcu_segcblist_insert_done_cbs()
382 WRITE_ONCE(rsclp->head, rclp->head); in rcu_segcblist_insert_done_cbs()
384 if (&rsclp->head == rsclp->tails[i]) in rcu_segcblist_insert_done_cbs()
385 WRITE_ONCE(rsclp->tails[i], rclp->tail); in rcu_segcblist_insert_done_cbs()
396 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, in rcu_segcblist_insert_pend_cbs() argument
401 WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head); in rcu_segcblist_insert_pend_cbs()
402 WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); in rcu_segcblist_insert_pend_cbs()
411 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) in rcu_segcblist_advance() argument
415 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); in rcu_segcblist_advance()
416 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) in rcu_segcblist_advance()
424 if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) in rcu_segcblist_advance()
426 WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); in rcu_segcblist_advance()
435 WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_advance()
444 if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) in rcu_segcblist_advance()
446 WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); in rcu_segcblist_advance()
447 rsclp->gp_seq[j] = rsclp->gp_seq[i]; in rcu_segcblist_advance()
466 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) in rcu_segcblist_accelerate() argument
470 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); in rcu_segcblist_accelerate()
471 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) in rcu_segcblist_accelerate()
483 if (rsclp->tails[i] != rsclp->tails[i - 1] && in rcu_segcblist_accelerate()
484 ULONG_CMP_LT(rsclp->gp_seq[i], seq)) in rcu_segcblist_accelerate()
511 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); in rcu_segcblist_accelerate()
512 rsclp->gp_seq[i] = seq; in rcu_segcblist_accelerate()