Lines Matching refs:s
41 static inline void __seqcount_init(seqcount_t *s, const char *name, in __seqcount_init() argument
47 lockdep_init_map(&s->dep_map, name, key, 0); in __seqcount_init()
48 s->sequence = 0; in __seqcount_init()
60 # define seqcount_init(s) \ argument
63 __seqcount_init((s), #s, &__key); \
66 static inline void seqcount_lockdep_reader_access(const seqcount_t *s) in seqcount_lockdep_reader_access() argument
68 seqcount_t *l = (seqcount_t *)s; in seqcount_lockdep_reader_access()
79 # define seqcount_init(s) __seqcount_init(s, NULL, NULL) argument
123 #define seqcount_LOCKNAME_init(s, _lock, lockname) \ argument
125 seqcount_##lockname##_t *____s = (s); \
130 #define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock) argument
131 #define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock) argument
132 #define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock) argument
133 #define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex) argument
146 __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s) \
148 return &s->seqcount; \
152 __seqprop_##lockname##_const_ptr(const seqcount_##lockname##_t *s) \
154 return &s->seqcount; \
158 __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s) \
160 unsigned seq = smp_load_acquire(&s->seqcount.sequence); \
166 __SEQ_LOCK(lockbase##_lock(s->lock)); \
167 __SEQ_LOCK(lockbase##_unlock(s->lock)); \
173 seq = smp_load_acquire(&s->seqcount.sequence); \
180 __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s) \
190 __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s) \
192 __SEQ_LOCK(lockdep_assert_held(s->lock)); \
199 static inline seqcount_t *__seqprop_ptr(seqcount_t *s) in __seqprop_ptr() argument
201 return s; in __seqprop_ptr()
204 static inline const seqcount_t *__seqprop_const_ptr(const seqcount_t *s) in __seqprop_const_ptr() argument
206 return s; in __seqprop_const_ptr()
209 static inline unsigned __seqprop_sequence(const seqcount_t *s) in __seqprop_sequence() argument
211 return smp_load_acquire(&s->sequence); in __seqprop_sequence()
214 static inline bool __seqprop_preemptible(const seqcount_t *s) in __seqprop_preemptible() argument
219 static inline void __seqprop_assert(const seqcount_t *s) in __seqprop_assert() argument
249 #define __seqprop_case(s, lockname, prop) \ argument
252 #define __seqprop(s, prop) _Generic(*(s), \ argument
254 __seqprop_case((s), raw_spinlock, prop), \
255 __seqprop_case((s), spinlock, prop), \
256 __seqprop_case((s), rwlock, prop), \
257 __seqprop_case((s), mutex, prop))
259 #define seqprop_ptr(s) __seqprop(s, ptr)(s) argument
260 #define seqprop_const_ptr(s) __seqprop(s, const_ptr)(s) argument
261 #define seqprop_sequence(s) __seqprop(s, sequence)(s) argument
262 #define seqprop_preemptible(s) __seqprop(s, preemptible)(s) argument
263 #define seqprop_assert(s) __seqprop(s, assert)(s) argument
271 #define __read_seqcount_begin(s) \ argument
275 while ((__seq = seqprop_sequence(s)) & 1) \
288 #define raw_read_seqcount_begin(s) __read_seqcount_begin(s) argument
296 #define read_seqcount_begin(s) \ argument
298 seqcount_lockdep_reader_access(seqprop_const_ptr(s)); \
299 raw_read_seqcount_begin(s); \
313 #define raw_read_seqcount(s) \ argument
315 unsigned __seq = seqprop_sequence(s); \
338 #define raw_seqcount_try_begin(s, start) \ argument
340 start = raw_read_seqcount(s); \
361 #define raw_seqcount_begin(s) \ argument
367 raw_read_seqcount(s) & ~1; \
385 #define __read_seqcount_retry(s, start) \ argument
386 do___read_seqcount_retry(seqprop_const_ptr(s), start)
388 static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start)
391 return unlikely(READ_ONCE(s->sequence) != start);
405 #define read_seqcount_retry(s, start) \ argument
406 do_read_seqcount_retry(seqprop_const_ptr(s), start)
408 static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start) in do_read_seqcount_retry() argument
411 return do___read_seqcount_retry(s, start); in do_read_seqcount_retry()
420 #define raw_write_seqcount_begin(s) \ argument
422 if (seqprop_preemptible(s)) \
425 do_raw_write_seqcount_begin(seqprop_ptr(s)); \
428 static inline void do_raw_write_seqcount_begin(seqcount_t *s) in do_raw_write_seqcount_begin() argument
431 s->sequence++; in do_raw_write_seqcount_begin()
441 #define raw_write_seqcount_end(s) \ argument
443 do_raw_write_seqcount_end(seqprop_ptr(s)); \
445 if (seqprop_preemptible(s)) \
449 static inline void do_raw_write_seqcount_end(seqcount_t *s) in do_raw_write_seqcount_end() argument
452 s->sequence++; in do_raw_write_seqcount_end()
465 #define write_seqcount_begin_nested(s, subclass) \ argument
467 seqprop_assert(s); \
469 if (seqprop_preemptible(s)) \
472 do_write_seqcount_begin_nested(seqprop_ptr(s), subclass); \
475 static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass) in do_write_seqcount_begin_nested() argument
477 seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); in do_write_seqcount_begin_nested()
478 do_raw_write_seqcount_begin(s); in do_write_seqcount_begin_nested()
491 #define write_seqcount_begin(s) \ argument
493 seqprop_assert(s); \
495 if (seqprop_preemptible(s)) \
498 do_write_seqcount_begin(seqprop_ptr(s)); \
501 static inline void do_write_seqcount_begin(seqcount_t *s) in do_write_seqcount_begin() argument
503 do_write_seqcount_begin_nested(s, 0); in do_write_seqcount_begin()
513 #define write_seqcount_end(s) \ argument
515 do_write_seqcount_end(seqprop_ptr(s)); \
517 if (seqprop_preemptible(s)) \
521 static inline void do_write_seqcount_end(seqcount_t *s) in do_write_seqcount_end() argument
523 seqcount_release(&s->dep_map, _RET_IP_); in do_write_seqcount_end()
524 do_raw_write_seqcount_end(s); in do_write_seqcount_end()
568 #define raw_write_seqcount_barrier(s) \ argument
569 do_raw_write_seqcount_barrier(seqprop_ptr(s))
571 static inline void do_raw_write_seqcount_barrier(seqcount_t *s) in do_raw_write_seqcount_barrier() argument
574 s->sequence++; in do_raw_write_seqcount_barrier()
576 s->sequence++; in do_raw_write_seqcount_barrier()
588 #define write_seqcount_invalidate(s) \ argument
589 do_write_seqcount_invalidate(seqprop_ptr(s))
591 static inline void do_write_seqcount_invalidate(seqcount_t *s) in do_write_seqcount_invalidate() argument
595 s->sequence+=2; in do_write_seqcount_invalidate()
625 #define seqcount_latch_init(s) seqcount_init(&(s)->seqcount) argument
638 static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) in raw_read_seqcount_latch() argument
644 return READ_ONCE(s->seqcount.sequence); in raw_read_seqcount_latch()
658 static __always_inline unsigned read_seqcount_latch(const seqcount_latch_t *s) in read_seqcount_latch() argument
661 return raw_read_seqcount_latch(s); in read_seqcount_latch()
672 raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) in raw_read_seqcount_latch_retry() argument
675 return unlikely(READ_ONCE(s->seqcount.sequence) != start); in raw_read_seqcount_latch_retry()
686 read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) in read_seqcount_latch_retry() argument
689 return raw_read_seqcount_latch_retry(s, start); in read_seqcount_latch_retry()
696 static __always_inline void raw_write_seqcount_latch(seqcount_latch_t *s) in raw_write_seqcount_latch() argument
699 s->seqcount.sequence++; in raw_write_seqcount_latch()
778 static __always_inline void write_seqcount_latch_begin(seqcount_latch_t *s) in write_seqcount_latch_begin() argument
781 raw_write_seqcount_latch(s); in write_seqcount_latch_begin()
788 static __always_inline void write_seqcount_latch(seqcount_latch_t *s) in write_seqcount_latch() argument
790 raw_write_seqcount_latch(s); in write_seqcount_latch()
800 static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s) in write_seqcount_latch_end() argument