1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _LINUX_KCSAN_CHECKS_H
4 #define _LINUX_KCSAN_CHECKS_H
5
6 /* Note: Only include what is already included by compiler.h. */
7 #include <linux/compiler_attributes.h>
8 #include <linux/types.h>
9
10 /* Access types -- if KCSAN_ACCESS_WRITE is not set, the access is a read. */
11 #define KCSAN_ACCESS_WRITE (1 << 0) /* Access is a write. */
12 #define KCSAN_ACCESS_COMPOUND (1 << 1) /* Compounded read-write instrumentation. */
13 #define KCSAN_ACCESS_ATOMIC (1 << 2) /* Access is atomic. */
14 /* The following are special, and never due to compiler instrumentation. */
15 #define KCSAN_ACCESS_ASSERT (1 << 3) /* Access is an assertion. */
16 #define KCSAN_ACCESS_SCOPED (1 << 4) /* Access is a scoped access. */
17
18 /*
19 * __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used
20 * even in compilation units that selectively disable KCSAN, but must use KCSAN
21 * to validate access to an address. Never use these in header files!
22 */
23 #ifdef CONFIG_KCSAN
24 /**
25 * __kcsan_check_access - check generic access for races
26 *
27 * @ptr: address of access
28 * @size: size of access
29 * @type: access type modifier
30 */
31 void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
32
33 /**
34 * kcsan_disable_current - disable KCSAN for the current context
35 *
36 * Supports nesting.
37 */
38 void kcsan_disable_current(void);
39
40 /**
41 * kcsan_enable_current - re-enable KCSAN for the current context
42 *
43 * Supports nesting.
44 */
45 void kcsan_enable_current(void);
46 void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */
47
48 /**
49 * kcsan_nestable_atomic_begin - begin nestable atomic region
50 *
51 * Accesses within the atomic region may appear to race with other accesses but
52 * should be considered atomic.
53 */
54 void kcsan_nestable_atomic_begin(void);
55
56 /**
57 * kcsan_nestable_atomic_end - end nestable atomic region
58 */
59 void kcsan_nestable_atomic_end(void);
60
61 /**
62 * kcsan_flat_atomic_begin - begin flat atomic region
63 *
64 * Accesses within the atomic region may appear to race with other accesses but
65 * should be considered atomic.
66 */
67 void kcsan_flat_atomic_begin(void);
68
69 /**
70 * kcsan_flat_atomic_end - end flat atomic region
71 */
72 void kcsan_flat_atomic_end(void);
73
74 /**
75 * kcsan_atomic_next - consider following accesses as atomic
76 *
77 * Force treating the next n memory accesses for the current context as atomic
78 * operations.
79 *
80 * @n: number of following memory accesses to treat as atomic.
81 */
82 void kcsan_atomic_next(int n);
83
84 /**
85 * kcsan_set_access_mask - set access mask
86 *
87 * Set the access mask for all accesses for the current context if non-zero.
88 * Only value changes to bits set in the mask will be reported.
89 *
90 * @mask: bitmask
91 */
92 void kcsan_set_access_mask(unsigned long mask);
93
94 /* Scoped access information. */
95 struct kcsan_scoped_access {
96 struct list_head list;
97 const volatile void *ptr;
98 size_t size;
99 int type;
100 };
101 /*
102 * Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes
103 * out of scope; relies on attribute "cleanup", which is supported by all
104 * compilers that support KCSAN.
105 */
106 #define __kcsan_cleanup_scoped \
107 __maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access)))
108
109 /**
110 * kcsan_begin_scoped_access - begin scoped access
111 *
112 * Begin scoped access and initialize @sa, which will cause KCSAN to
113 * continuously check the memory range in the current thread until
114 * kcsan_end_scoped_access() is called for @sa.
115 *
116 * Scoped accesses are implemented by appending @sa to an internal list for the
117 * current execution context, and then checked on every call into the KCSAN
118 * runtime.
119 *
120 * @ptr: address of access
121 * @size: size of access
122 * @type: access type modifier
123 * @sa: struct kcsan_scoped_access to use for the scope of the access
124 */
125 struct kcsan_scoped_access *
126 kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
127 struct kcsan_scoped_access *sa);
128
129 /**
130 * kcsan_end_scoped_access - end scoped access
131 *
132 * End a scoped access, which will stop KCSAN checking the memory range.
133 * Requires that kcsan_begin_scoped_access() was previously called once for @sa.
134 *
135 * @sa: a previously initialized struct kcsan_scoped_access
136 */
137 void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);
138
139
140 #else /* CONFIG_KCSAN */
141
__kcsan_check_access(const volatile void * ptr,size_t size,int type)142 static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
143 int type) { }
144
kcsan_disable_current(void)145 static inline void kcsan_disable_current(void) { }
kcsan_enable_current(void)146 static inline void kcsan_enable_current(void) { }
kcsan_enable_current_nowarn(void)147 static inline void kcsan_enable_current_nowarn(void) { }
kcsan_nestable_atomic_begin(void)148 static inline void kcsan_nestable_atomic_begin(void) { }
kcsan_nestable_atomic_end(void)149 static inline void kcsan_nestable_atomic_end(void) { }
kcsan_flat_atomic_begin(void)150 static inline void kcsan_flat_atomic_begin(void) { }
kcsan_flat_atomic_end(void)151 static inline void kcsan_flat_atomic_end(void) { }
kcsan_atomic_next(int n)152 static inline void kcsan_atomic_next(int n) { }
kcsan_set_access_mask(unsigned long mask)153 static inline void kcsan_set_access_mask(unsigned long mask) { }
154
155 struct kcsan_scoped_access { };
156 #define __kcsan_cleanup_scoped __maybe_unused
157 static inline struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void * ptr,size_t size,int type,struct kcsan_scoped_access * sa)158 kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
159 struct kcsan_scoped_access *sa) { return sa; }
kcsan_end_scoped_access(struct kcsan_scoped_access * sa)160 static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }
161
162 #endif /* CONFIG_KCSAN */
163
164 #ifdef __SANITIZE_THREAD__
165 /*
166 * Only calls into the runtime when the particular compilation unit has KCSAN
167 * instrumentation enabled. May be used in header files.
168 */
169 #define kcsan_check_access __kcsan_check_access
170
171 /*
172 * Only use these to disable KCSAN for accesses in the current compilation unit;
173 * calls into libraries may still perform KCSAN checks.
174 */
175 #define __kcsan_disable_current kcsan_disable_current
176 #define __kcsan_enable_current kcsan_enable_current_nowarn
177 #else
kcsan_check_access(const volatile void * ptr,size_t size,int type)178 static inline void kcsan_check_access(const volatile void *ptr, size_t size,
179 int type) { }
__kcsan_enable_current(void)180 static inline void __kcsan_enable_current(void) { }
__kcsan_disable_current(void)181 static inline void __kcsan_disable_current(void) { }
182 #endif
183
184 /**
185 * __kcsan_check_read - check regular read access for races
186 *
187 * @ptr: address of access
188 * @size: size of access
189 */
190 #define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
191
192 /**
193 * __kcsan_check_write - check regular write access for races
194 *
195 * @ptr: address of access
196 * @size: size of access
197 */
198 #define __kcsan_check_write(ptr, size) \
199 __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
200
201 /**
202 * __kcsan_check_read_write - check regular read-write access for races
203 *
204 * @ptr: address of access
205 * @size: size of access
206 */
207 #define __kcsan_check_read_write(ptr, size) \
208 __kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
209
210 /**
211 * kcsan_check_read - check regular read access for races
212 *
213 * @ptr: address of access
214 * @size: size of access
215 */
216 #define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
217
218 /**
219 * kcsan_check_write - check regular write access for races
220 *
221 * @ptr: address of access
222 * @size: size of access
223 */
224 #define kcsan_check_write(ptr, size) \
225 kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
226
227 /**
228 * kcsan_check_read_write - check regular read-write access for races
229 *
230 * @ptr: address of access
231 * @size: size of access
232 */
233 #define kcsan_check_read_write(ptr, size) \
234 kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
235
236 /*
237 * Check for atomic accesses: if atomic accesses are not ignored, this simply
238 * aliases to kcsan_check_access(), otherwise becomes a no-op.
239 */
240 #ifdef CONFIG_KCSAN_IGNORE_ATOMICS
241 #define kcsan_check_atomic_read(...) do { } while (0)
242 #define kcsan_check_atomic_write(...) do { } while (0)
243 #define kcsan_check_atomic_read_write(...) do { } while (0)
244 #else
245 #define kcsan_check_atomic_read(ptr, size) \
246 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
247 #define kcsan_check_atomic_write(ptr, size) \
248 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
249 #define kcsan_check_atomic_read_write(ptr, size) \
250 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND)
251 #endif
252
253 /**
254 * ASSERT_EXCLUSIVE_WRITER - assert no concurrent writes to @var
255 *
256 * Assert that there are no concurrent writes to @var; other readers are
257 * allowed. This assertion can be used to specify properties of concurrent code,
258 * where violation cannot be detected as a normal data race.
259 *
260 * For example, if we only have a single writer, but multiple concurrent
261 * readers, to avoid data races, all these accesses must be marked; even
262 * concurrent marked writes racing with the single writer are bugs.
263 * Unfortunately, due to being marked, they are no longer data races. For cases
264 * like these, we can use the macro as follows:
265 *
266 * .. code-block:: c
267 *
268 * void writer(void) {
269 * spin_lock(&update_foo_lock);
270 * ASSERT_EXCLUSIVE_WRITER(shared_foo);
271 * WRITE_ONCE(shared_foo, ...);
272 * spin_unlock(&update_foo_lock);
273 * }
274 * void reader(void) {
275 * // update_foo_lock does not need to be held!
276 * ... = READ_ONCE(shared_foo);
277 * }
278 *
279 * Note: ASSERT_EXCLUSIVE_WRITER_SCOPED(), if applicable, performs more thorough
280 * checking if a clear scope where no concurrent writes are expected exists.
281 *
282 * @var: variable to assert on
283 */
284 #define ASSERT_EXCLUSIVE_WRITER(var) \
285 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
286
287 /*
288 * Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is
289 * expected to be unique for the scope in which instances of kcsan_scoped_access
290 * are declared.
291 */
292 #define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix
293 #define __ASSERT_EXCLUSIVE_SCOPED(var, type, id) \
294 struct kcsan_scoped_access __kcsan_scoped_name(id, _) \
295 __kcsan_cleanup_scoped; \
296 struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p) \
297 __maybe_unused = kcsan_begin_scoped_access( \
298 &(var), sizeof(var), KCSAN_ACCESS_SCOPED | (type), \
299 &__kcsan_scoped_name(id, _))
300
301 /**
302 * ASSERT_EXCLUSIVE_WRITER_SCOPED - assert no concurrent writes to @var in scope
303 *
304 * Scoped variant of ASSERT_EXCLUSIVE_WRITER().
305 *
306 * Assert that there are no concurrent writes to @var for the duration of the
307 * scope in which it is introduced. This provides a better way to fully cover
308 * the enclosing scope, compared to multiple ASSERT_EXCLUSIVE_WRITER(), and
309 * increases the likelihood for KCSAN to detect racing accesses.
310 *
311 * For example, it allows finding race-condition bugs that only occur due to
312 * state changes within the scope itself:
313 *
314 * .. code-block:: c
315 *
316 * void writer(void) {
317 * spin_lock(&update_foo_lock);
318 * {
319 * ASSERT_EXCLUSIVE_WRITER_SCOPED(shared_foo);
320 * WRITE_ONCE(shared_foo, 42);
321 * ...
322 * // shared_foo should still be 42 here!
323 * }
324 * spin_unlock(&update_foo_lock);
325 * }
326 * void buggy(void) {
327 * if (READ_ONCE(shared_foo) == 42)
328 * WRITE_ONCE(shared_foo, 1); // bug!
329 * }
330 *
331 * @var: variable to assert on
332 */
333 #define ASSERT_EXCLUSIVE_WRITER_SCOPED(var) \
334 __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__)
335
336 /**
337 * ASSERT_EXCLUSIVE_ACCESS - assert no concurrent accesses to @var
338 *
339 * Assert that there are no concurrent accesses to @var (no readers nor
340 * writers). This assertion can be used to specify properties of concurrent
341 * code, where violation cannot be detected as a normal data race.
342 *
343 * For example, where exclusive access is expected after determining no other
344 * users of an object are left, but the object is not actually freed. We can
345 * check that this property actually holds as follows:
346 *
347 * .. code-block:: c
348 *
349 * if (refcount_dec_and_test(&obj->refcnt)) {
350 * ASSERT_EXCLUSIVE_ACCESS(*obj);
351 * do_some_cleanup(obj);
352 * release_for_reuse(obj);
353 * }
354 *
355 * Note:
356 *
357 * 1. ASSERT_EXCLUSIVE_ACCESS_SCOPED(), if applicable, performs more thorough
358 * checking if a clear scope where no concurrent accesses are expected exists.
359 *
360 * 2. For cases where the object is freed, `KASAN <kasan.html>`_ is a better
361 * fit to detect use-after-free bugs.
362 *
363 * @var: variable to assert on
364 */
365 #define ASSERT_EXCLUSIVE_ACCESS(var) \
366 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT)
367
368 /**
369 * ASSERT_EXCLUSIVE_ACCESS_SCOPED - assert no concurrent accesses to @var in scope
370 *
371 * Scoped variant of ASSERT_EXCLUSIVE_ACCESS().
372 *
373 * Assert that there are no concurrent accesses to @var (no readers nor writers)
374 * for the entire duration of the scope in which it is introduced. This provides
375 * a better way to fully cover the enclosing scope, compared to multiple
376 * ASSERT_EXCLUSIVE_ACCESS(), and increases the likelihood for KCSAN to detect
377 * racing accesses.
378 *
379 * @var: variable to assert on
380 */
381 #define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var) \
382 __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT, __COUNTER__)
383
384 /**
385 * ASSERT_EXCLUSIVE_BITS - assert no concurrent writes to subset of bits in @var
386 *
387 * Bit-granular variant of ASSERT_EXCLUSIVE_WRITER().
388 *
389 * Assert that there are no concurrent writes to a subset of bits in @var;
390 * concurrent readers are permitted. This assertion captures more detailed
391 * bit-level properties, compared to the other (word granularity) assertions.
392 * Only the bits set in @mask are checked for concurrent modifications, while
393 * ignoring the remaining bits, i.e. concurrent writes (or reads) to ~mask bits
394 * are ignored.
395 *
396 * Use this for variables, where some bits must not be modified concurrently,
397 * yet other bits are expected to be modified concurrently.
398 *
399 * For example, variables where, after initialization, some bits are read-only,
400 * but other bits may still be modified concurrently. A reader may wish to
401 * assert that this is true as follows:
402 *
403 * .. code-block:: c
404 *
405 * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
406 * foo = (READ_ONCE(flags) & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
407 *
408 * Note: The access that immediately follows ASSERT_EXCLUSIVE_BITS() is assumed
409 * to access the masked bits only, and KCSAN optimistically assumes it is
410 * therefore safe, even in the presence of data races, and marking it with
411 * READ_ONCE() is optional from KCSAN's point-of-view. We caution, however, that
412 * it may still be advisable to do so, since we cannot reason about all compiler
413 * optimizations when it comes to bit manipulations (on the reader and writer
414 * side). If you are sure nothing can go wrong, we can write the above simply
415 * as:
416 *
417 * .. code-block:: c
418 *
419 * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK);
420 * foo = (flags & READ_ONLY_MASK) >> READ_ONLY_SHIFT;
421 *
422 * Another example, where this may be used, is when certain bits of @var may
423 * only be modified when holding the appropriate lock, but other bits may still
424 * be modified concurrently. Writers, where other bits may change concurrently,
425 * could use the assertion as follows:
426 *
427 * .. code-block:: c
428 *
429 * spin_lock(&foo_lock);
430 * ASSERT_EXCLUSIVE_BITS(flags, FOO_MASK);
431 * old_flags = flags;
432 * new_flags = (old_flags & ~FOO_MASK) | (new_foo << FOO_SHIFT);
433 * if (cmpxchg(&flags, old_flags, new_flags) != old_flags) { ... }
434 * spin_unlock(&foo_lock);
435 *
436 * @var: variable to assert on
437 * @mask: only check for modifications to bits set in @mask
438 */
439 #define ASSERT_EXCLUSIVE_BITS(var, mask) \
440 do { \
441 kcsan_set_access_mask(mask); \
442 __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\
443 kcsan_set_access_mask(0); \
444 kcsan_atomic_next(1); \
445 } while (0)
446
447 #endif /* _LINUX_KCSAN_CHECKS_H */
448