1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KCSAN test with various race scenarious to test runtime behaviour. Since the
4 * interface with which KCSAN's reports are obtained is via the console, this is
5 * the output we should verify. For each test case checks the presence (or
6 * absence) of generated reports. Relies on 'console' tracepoint to capture
7 * reports as they appear in the kernel log.
8 *
9 * Makes use of KUnit for test organization, and the Torture framework for test
10 * thread control.
11 *
12 * Copyright (C) 2020, Google LLC.
13 * Author: Marco Elver <elver@google.com>
14 */
15
16 #include <kunit/test.h>
17 #include <linux/jiffies.h>
18 #include <linux/kcsan-checks.h>
19 #include <linux/kernel.h>
20 #include <linux/sched.h>
21 #include <linux/seqlock.h>
22 #include <linux/spinlock.h>
23 #include <linux/string.h>
24 #include <linux/timer.h>
25 #include <linux/torture.h>
26 #include <linux/tracepoint.h>
27 #include <linux/types.h>
28 #include <trace/events/printk.h>
29
30 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
31 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
32 #else
33 #define __KCSAN_ACCESS_RW(alt) (alt)
34 #endif
35
36 /* Points to current test-case memory access "kernels". */
37 static void (*access_kernels[2])(void);
38
39 static struct task_struct **threads; /* Lists of threads. */
40 static unsigned long end_time; /* End time of test. */
41
42 /* Report as observed from console. */
43 static struct {
44 spinlock_t lock;
45 int nlines;
46 char lines[3][512];
47 } observed = {
48 .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
49 };
50
51 /* Setup test checking loop. */
52 static __no_kcsan inline void
begin_test_checks(void (* func1)(void),void (* func2)(void))53 begin_test_checks(void (*func1)(void), void (*func2)(void))
54 {
55 kcsan_disable_current();
56
57 /*
58 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
59 * least one race is reported.
60 */
61 end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
62
63 /* Signal start; release potential initialization of shared data. */
64 smp_store_release(&access_kernels[0], func1);
65 smp_store_release(&access_kernels[1], func2);
66 }
67
68 /* End test checking loop. */
69 static __no_kcsan inline bool
end_test_checks(bool stop)70 end_test_checks(bool stop)
71 {
72 if (!stop && time_before(jiffies, end_time)) {
73 /* Continue checking */
74 might_sleep();
75 return false;
76 }
77
78 kcsan_enable_current();
79 return true;
80 }
81
82 /*
83 * Probe for console output: checks if a race was reported, and obtains observed
84 * lines of interest.
85 */
86 __no_kcsan
probe_console(void * ignore,const char * buf,size_t len)87 static void probe_console(void *ignore, const char *buf, size_t len)
88 {
89 unsigned long flags;
90 int nlines;
91
92 /*
93 * Note that KCSAN reports under a global lock, so we do not risk the
94 * possibility of having multiple reports interleaved. If that were the
95 * case, we'd expect tests to fail.
96 */
97
98 spin_lock_irqsave(&observed.lock, flags);
99 nlines = observed.nlines;
100
101 if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
102 /*
103 * KCSAN report and related to the test.
104 *
105 * The provided @buf is not NUL-terminated; copy no more than
106 * @len bytes and let strscpy() add the missing NUL-terminator.
107 */
108 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
109 nlines = 1;
110 } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
111 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
112
113 if (strnstr(buf, "race at unknown origin", len)) {
114 if (WARN_ON(nlines != 2))
115 goto out;
116
117 /* No second line of interest. */
118 strcpy(observed.lines[nlines++], "<none>");
119 }
120 }
121
122 out:
123 WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
124 spin_unlock_irqrestore(&observed.lock, flags);
125 }
126
127 /* Check if a report related to the test exists. */
128 __no_kcsan
report_available(void)129 static bool report_available(void)
130 {
131 return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
132 }
133
134 /* Report information we expect in a report. */
135 struct expect_report {
136 /* Access information of both accesses. */
137 struct {
138 void *fn; /* Function pointer to expected function of top frame. */
139 void *addr; /* Address of access; unchecked if NULL. */
140 size_t size; /* Size of access; unchecked if @addr is NULL. */
141 int type; /* Access type, see KCSAN_ACCESS definitions. */
142 } access[2];
143 };
144
145 /* Check observed report matches information in @r. */
146 __no_kcsan
report_matches(const struct expect_report * r)147 static bool report_matches(const struct expect_report *r)
148 {
149 const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
150 bool ret = false;
151 unsigned long flags;
152 typeof(*observed.lines) *expect;
153 const char *end;
154 char *cur;
155 int i;
156
157 /* Doubled-checked locking. */
158 if (!report_available())
159 return false;
160
161 expect = kmalloc(sizeof(observed.lines), GFP_KERNEL);
162 if (WARN_ON(!expect))
163 return false;
164
165 /* Generate expected report contents. */
166
167 /* Title */
168 cur = expect[0];
169 end = &expect[0][sizeof(expect[0]) - 1];
170 cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
171 is_assert ? "assert: race" : "data-race");
172 if (r->access[1].fn) {
173 char tmp[2][64];
174 int cmp;
175
176 /* Expect lexographically sorted function names in title. */
177 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
178 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
179 cmp = strcmp(tmp[0], tmp[1]);
180 cur += scnprintf(cur, end - cur, "%ps / %ps",
181 cmp < 0 ? r->access[0].fn : r->access[1].fn,
182 cmp < 0 ? r->access[1].fn : r->access[0].fn);
183 } else {
184 scnprintf(cur, end - cur, "%pS", r->access[0].fn);
185 /* The exact offset won't match, remove it. */
186 cur = strchr(expect[0], '+');
187 if (cur)
188 *cur = '\0';
189 }
190
191 /* Access 1 */
192 cur = expect[1];
193 end = &expect[1][sizeof(expect[1]) - 1];
194 if (!r->access[1].fn)
195 cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
196
197 /* Access 1 & 2 */
198 for (i = 0; i < 2; ++i) {
199 const int ty = r->access[i].type;
200 const char *const access_type =
201 (ty & KCSAN_ACCESS_ASSERT) ?
202 ((ty & KCSAN_ACCESS_WRITE) ?
203 "assert no accesses" :
204 "assert no writes") :
205 ((ty & KCSAN_ACCESS_WRITE) ?
206 ((ty & KCSAN_ACCESS_COMPOUND) ?
207 "read-write" :
208 "write") :
209 "read");
210 const char *const access_type_aux =
211 (ty & KCSAN_ACCESS_ATOMIC) ?
212 " (marked)" :
213 ((ty & KCSAN_ACCESS_SCOPED) ? " (scoped)" : "");
214
215 if (i == 1) {
216 /* Access 2 */
217 cur = expect[2];
218 end = &expect[2][sizeof(expect[2]) - 1];
219
220 if (!r->access[1].fn) {
221 /* Dummy string if no second access is available. */
222 strcpy(cur, "<none>");
223 break;
224 }
225 }
226
227 cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
228 access_type_aux);
229
230 if (r->access[i].addr) /* Address is optional. */
231 cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
232 r->access[i].addr, r->access[i].size);
233 }
234
235 spin_lock_irqsave(&observed.lock, flags);
236 if (!report_available())
237 goto out; /* A new report is being captured. */
238
239 /* Finally match expected output to what we actually observed. */
240 ret = strstr(observed.lines[0], expect[0]) &&
241 /* Access info may appear in any order. */
242 ((strstr(observed.lines[1], expect[1]) &&
243 strstr(observed.lines[2], expect[2])) ||
244 (strstr(observed.lines[1], expect[2]) &&
245 strstr(observed.lines[2], expect[1])));
246 out:
247 spin_unlock_irqrestore(&observed.lock, flags);
248 kfree(expect);
249 return ret;
250 }
251
252 /* ===== Test kernels ===== */
253
254 static long test_sink;
255 static long test_var;
256 /* @test_array should be large enough to fall into multiple watchpoint slots. */
257 static long test_array[3 * PAGE_SIZE / sizeof(long)];
258 static struct {
259 long val[8];
260 } test_struct;
261 static DEFINE_SEQLOCK(test_seqlock);
262
263 /*
264 * Helper to avoid compiler optimizing out reads, and to generate source values
265 * for writes.
266 */
267 __no_kcsan
sink_value(long v)268 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
269
test_kernel_read(void)270 static noinline void test_kernel_read(void) { sink_value(test_var); }
271
test_kernel_write(void)272 static noinline void test_kernel_write(void)
273 {
274 test_var = READ_ONCE_NOCHECK(test_sink) + 1;
275 }
276
test_kernel_write_nochange(void)277 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
278
279 /* Suffixed by value-change exception filter. */
test_kernel_write_nochange_rcu(void)280 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
281
test_kernel_read_atomic(void)282 static noinline void test_kernel_read_atomic(void)
283 {
284 sink_value(READ_ONCE(test_var));
285 }
286
test_kernel_write_atomic(void)287 static noinline void test_kernel_write_atomic(void)
288 {
289 WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
290 }
291
test_kernel_atomic_rmw(void)292 static noinline void test_kernel_atomic_rmw(void)
293 {
294 /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
295 __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
296 }
297
298 __no_kcsan
test_kernel_write_uninstrumented(void)299 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
300
test_kernel_data_race(void)301 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
302
test_kernel_assert_writer(void)303 static noinline void test_kernel_assert_writer(void)
304 {
305 ASSERT_EXCLUSIVE_WRITER(test_var);
306 }
307
test_kernel_assert_access(void)308 static noinline void test_kernel_assert_access(void)
309 {
310 ASSERT_EXCLUSIVE_ACCESS(test_var);
311 }
312
313 #define TEST_CHANGE_BITS 0xff00ff00
314
test_kernel_change_bits(void)315 static noinline void test_kernel_change_bits(void)
316 {
317 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
318 /*
319 * Avoid race of unknown origin for this test, just pretend they
320 * are atomic.
321 */
322 kcsan_nestable_atomic_begin();
323 test_var ^= TEST_CHANGE_BITS;
324 kcsan_nestable_atomic_end();
325 } else
326 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
327 }
328
test_kernel_assert_bits_change(void)329 static noinline void test_kernel_assert_bits_change(void)
330 {
331 ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
332 }
333
test_kernel_assert_bits_nochange(void)334 static noinline void test_kernel_assert_bits_nochange(void)
335 {
336 ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
337 }
338
339 /* To check that scoped assertions do trigger anywhere in scope. */
test_enter_scope(void)340 static noinline void test_enter_scope(void)
341 {
342 int x = 0;
343
344 /* Unrelated accesses to scoped assert. */
345 READ_ONCE(test_sink);
346 kcsan_check_read(&x, sizeof(x));
347 }
348
test_kernel_assert_writer_scoped(void)349 static noinline void test_kernel_assert_writer_scoped(void)
350 {
351 ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
352 test_enter_scope();
353 }
354
test_kernel_assert_access_scoped(void)355 static noinline void test_kernel_assert_access_scoped(void)
356 {
357 ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
358 test_enter_scope();
359 }
360
test_kernel_rmw_array(void)361 static noinline void test_kernel_rmw_array(void)
362 {
363 int i;
364
365 for (i = 0; i < ARRAY_SIZE(test_array); ++i)
366 test_array[i]++;
367 }
368
test_kernel_write_struct(void)369 static noinline void test_kernel_write_struct(void)
370 {
371 kcsan_check_write(&test_struct, sizeof(test_struct));
372 kcsan_disable_current();
373 test_struct.val[3]++; /* induce value change */
374 kcsan_enable_current();
375 }
376
test_kernel_write_struct_part(void)377 static noinline void test_kernel_write_struct_part(void)
378 {
379 test_struct.val[3] = 42;
380 }
381
test_kernel_read_struct_zero_size(void)382 static noinline void test_kernel_read_struct_zero_size(void)
383 {
384 kcsan_check_read(&test_struct.val[3], 0);
385 }
386
test_kernel_jiffies_reader(void)387 static noinline void test_kernel_jiffies_reader(void)
388 {
389 sink_value((long)jiffies);
390 }
391
test_kernel_seqlock_reader(void)392 static noinline void test_kernel_seqlock_reader(void)
393 {
394 unsigned int seq;
395
396 do {
397 seq = read_seqbegin(&test_seqlock);
398 sink_value(test_var);
399 } while (read_seqretry(&test_seqlock, seq));
400 }
401
test_kernel_seqlock_writer(void)402 static noinline void test_kernel_seqlock_writer(void)
403 {
404 unsigned long flags;
405
406 write_seqlock_irqsave(&test_seqlock, flags);
407 test_var++;
408 write_sequnlock_irqrestore(&test_seqlock, flags);
409 }
410
test_kernel_atomic_builtins(void)411 static noinline void test_kernel_atomic_builtins(void)
412 {
413 /*
414 * Generate concurrent accesses, expecting no reports, ensuring KCSAN
415 * treats builtin atomics as actually atomic.
416 */
417 __atomic_load_n(&test_var, __ATOMIC_RELAXED);
418 }
419
420 /* ===== Test cases ===== */
421
422 /* Simple test with normal data race. */
423 __no_kcsan
test_basic(struct kunit * test)424 static void test_basic(struct kunit *test)
425 {
426 const struct expect_report expect = {
427 .access = {
428 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
429 { test_kernel_read, &test_var, sizeof(test_var), 0 },
430 },
431 };
432 static const struct expect_report never = {
433 .access = {
434 { test_kernel_read, &test_var, sizeof(test_var), 0 },
435 { test_kernel_read, &test_var, sizeof(test_var), 0 },
436 },
437 };
438 bool match_expect = false;
439 bool match_never = false;
440
441 begin_test_checks(test_kernel_write, test_kernel_read);
442 do {
443 match_expect |= report_matches(&expect);
444 match_never = report_matches(&never);
445 } while (!end_test_checks(match_never));
446 KUNIT_EXPECT_TRUE(test, match_expect);
447 KUNIT_EXPECT_FALSE(test, match_never);
448 }
449
450 /*
451 * Stress KCSAN with lots of concurrent races on different addresses until
452 * timeout.
453 */
454 __no_kcsan
test_concurrent_races(struct kunit * test)455 static void test_concurrent_races(struct kunit *test)
456 {
457 const struct expect_report expect = {
458 .access = {
459 /* NULL will match any address. */
460 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
461 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
462 },
463 };
464 static const struct expect_report never = {
465 .access = {
466 { test_kernel_rmw_array, NULL, 0, 0 },
467 { test_kernel_rmw_array, NULL, 0, 0 },
468 },
469 };
470 bool match_expect = false;
471 bool match_never = false;
472
473 begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
474 do {
475 match_expect |= report_matches(&expect);
476 match_never |= report_matches(&never);
477 } while (!end_test_checks(false));
478 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
479 KUNIT_EXPECT_FALSE(test, match_never);
480 }
481
482 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
483 __no_kcsan
test_novalue_change(struct kunit * test)484 static void test_novalue_change(struct kunit *test)
485 {
486 const struct expect_report expect = {
487 .access = {
488 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
489 { test_kernel_read, &test_var, sizeof(test_var), 0 },
490 },
491 };
492 bool match_expect = false;
493
494 begin_test_checks(test_kernel_write_nochange, test_kernel_read);
495 do {
496 match_expect = report_matches(&expect);
497 } while (!end_test_checks(match_expect));
498 if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
499 KUNIT_EXPECT_FALSE(test, match_expect);
500 else
501 KUNIT_EXPECT_TRUE(test, match_expect);
502 }
503
504 /*
505 * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
506 * never apply work.
507 */
508 __no_kcsan
test_novalue_change_exception(struct kunit * test)509 static void test_novalue_change_exception(struct kunit *test)
510 {
511 const struct expect_report expect = {
512 .access = {
513 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
514 { test_kernel_read, &test_var, sizeof(test_var), 0 },
515 },
516 };
517 bool match_expect = false;
518
519 begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
520 do {
521 match_expect = report_matches(&expect);
522 } while (!end_test_checks(match_expect));
523 KUNIT_EXPECT_TRUE(test, match_expect);
524 }
525
526 /* Test that data races of unknown origin are reported. */
527 __no_kcsan
test_unknown_origin(struct kunit * test)528 static void test_unknown_origin(struct kunit *test)
529 {
530 const struct expect_report expect = {
531 .access = {
532 { test_kernel_read, &test_var, sizeof(test_var), 0 },
533 { NULL },
534 },
535 };
536 bool match_expect = false;
537
538 begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
539 do {
540 match_expect = report_matches(&expect);
541 } while (!end_test_checks(match_expect));
542 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
543 KUNIT_EXPECT_TRUE(test, match_expect);
544 else
545 KUNIT_EXPECT_FALSE(test, match_expect);
546 }
547
548 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
549 __no_kcsan
test_write_write_assume_atomic(struct kunit * test)550 static void test_write_write_assume_atomic(struct kunit *test)
551 {
552 const struct expect_report expect = {
553 .access = {
554 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
555 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
556 },
557 };
558 bool match_expect = false;
559
560 begin_test_checks(test_kernel_write, test_kernel_write);
561 do {
562 sink_value(READ_ONCE(test_var)); /* induce value-change */
563 match_expect = report_matches(&expect);
564 } while (!end_test_checks(match_expect));
565 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
566 KUNIT_EXPECT_FALSE(test, match_expect);
567 else
568 KUNIT_EXPECT_TRUE(test, match_expect);
569 }
570
571 /*
572 * Test that data races with writes larger than word-size are always reported,
573 * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
574 */
575 __no_kcsan
test_write_write_struct(struct kunit * test)576 static void test_write_write_struct(struct kunit *test)
577 {
578 const struct expect_report expect = {
579 .access = {
580 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
581 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
582 },
583 };
584 bool match_expect = false;
585
586 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
587 do {
588 match_expect = report_matches(&expect);
589 } while (!end_test_checks(match_expect));
590 KUNIT_EXPECT_TRUE(test, match_expect);
591 }
592
593 /*
594 * Test that data races where only one write is larger than word-size are always
595 * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
596 */
597 __no_kcsan
test_write_write_struct_part(struct kunit * test)598 static void test_write_write_struct_part(struct kunit *test)
599 {
600 const struct expect_report expect = {
601 .access = {
602 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
603 { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
604 },
605 };
606 bool match_expect = false;
607
608 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
609 do {
610 match_expect = report_matches(&expect);
611 } while (!end_test_checks(match_expect));
612 KUNIT_EXPECT_TRUE(test, match_expect);
613 }
614
615 /* Test that races with atomic accesses never result in reports. */
616 __no_kcsan
test_read_atomic_write_atomic(struct kunit * test)617 static void test_read_atomic_write_atomic(struct kunit *test)
618 {
619 bool match_never = false;
620
621 begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
622 do {
623 match_never = report_available();
624 } while (!end_test_checks(match_never));
625 KUNIT_EXPECT_FALSE(test, match_never);
626 }
627
628 /* Test that a race with an atomic and plain access result in reports. */
629 __no_kcsan
test_read_plain_atomic_write(struct kunit * test)630 static void test_read_plain_atomic_write(struct kunit *test)
631 {
632 const struct expect_report expect = {
633 .access = {
634 { test_kernel_read, &test_var, sizeof(test_var), 0 },
635 { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
636 },
637 };
638 bool match_expect = false;
639
640 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
641 return;
642
643 begin_test_checks(test_kernel_read, test_kernel_write_atomic);
644 do {
645 match_expect = report_matches(&expect);
646 } while (!end_test_checks(match_expect));
647 KUNIT_EXPECT_TRUE(test, match_expect);
648 }
649
650 /* Test that atomic RMWs generate correct report. */
651 __no_kcsan
test_read_plain_atomic_rmw(struct kunit * test)652 static void test_read_plain_atomic_rmw(struct kunit *test)
653 {
654 const struct expect_report expect = {
655 .access = {
656 { test_kernel_read, &test_var, sizeof(test_var), 0 },
657 { test_kernel_atomic_rmw, &test_var, sizeof(test_var),
658 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
659 },
660 };
661 bool match_expect = false;
662
663 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
664 return;
665
666 begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
667 do {
668 match_expect = report_matches(&expect);
669 } while (!end_test_checks(match_expect));
670 KUNIT_EXPECT_TRUE(test, match_expect);
671 }
672
673 /* Zero-sized accesses should never cause data race reports. */
674 __no_kcsan
test_zero_size_access(struct kunit * test)675 static void test_zero_size_access(struct kunit *test)
676 {
677 const struct expect_report expect = {
678 .access = {
679 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
680 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
681 },
682 };
683 const struct expect_report never = {
684 .access = {
685 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
686 { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
687 },
688 };
689 bool match_expect = false;
690 bool match_never = false;
691
692 begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
693 do {
694 match_expect |= report_matches(&expect);
695 match_never = report_matches(&never);
696 } while (!end_test_checks(match_never));
697 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
698 KUNIT_EXPECT_FALSE(test, match_never);
699 }
700
701 /* Test the data_race() macro. */
702 __no_kcsan
test_data_race(struct kunit * test)703 static void test_data_race(struct kunit *test)
704 {
705 bool match_never = false;
706
707 begin_test_checks(test_kernel_data_race, test_kernel_data_race);
708 do {
709 match_never = report_available();
710 } while (!end_test_checks(match_never));
711 KUNIT_EXPECT_FALSE(test, match_never);
712 }
713
714 __no_kcsan
test_assert_exclusive_writer(struct kunit * test)715 static void test_assert_exclusive_writer(struct kunit *test)
716 {
717 const struct expect_report expect = {
718 .access = {
719 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
720 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
721 },
722 };
723 bool match_expect = false;
724
725 begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
726 do {
727 match_expect = report_matches(&expect);
728 } while (!end_test_checks(match_expect));
729 KUNIT_EXPECT_TRUE(test, match_expect);
730 }
731
732 __no_kcsan
test_assert_exclusive_access(struct kunit * test)733 static void test_assert_exclusive_access(struct kunit *test)
734 {
735 const struct expect_report expect = {
736 .access = {
737 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
738 { test_kernel_read, &test_var, sizeof(test_var), 0 },
739 },
740 };
741 bool match_expect = false;
742
743 begin_test_checks(test_kernel_assert_access, test_kernel_read);
744 do {
745 match_expect = report_matches(&expect);
746 } while (!end_test_checks(match_expect));
747 KUNIT_EXPECT_TRUE(test, match_expect);
748 }
749
750 __no_kcsan
test_assert_exclusive_access_writer(struct kunit * test)751 static void test_assert_exclusive_access_writer(struct kunit *test)
752 {
753 const struct expect_report expect_access_writer = {
754 .access = {
755 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
756 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
757 },
758 };
759 const struct expect_report expect_access_access = {
760 .access = {
761 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
762 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
763 },
764 };
765 const struct expect_report never = {
766 .access = {
767 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
768 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
769 },
770 };
771 bool match_expect_access_writer = false;
772 bool match_expect_access_access = false;
773 bool match_never = false;
774
775 begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
776 do {
777 match_expect_access_writer |= report_matches(&expect_access_writer);
778 match_expect_access_access |= report_matches(&expect_access_access);
779 match_never |= report_matches(&never);
780 } while (!end_test_checks(match_never));
781 KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
782 KUNIT_EXPECT_TRUE(test, match_expect_access_access);
783 KUNIT_EXPECT_FALSE(test, match_never);
784 }
785
786 __no_kcsan
test_assert_exclusive_bits_change(struct kunit * test)787 static void test_assert_exclusive_bits_change(struct kunit *test)
788 {
789 const struct expect_report expect = {
790 .access = {
791 { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
792 { test_kernel_change_bits, &test_var, sizeof(test_var),
793 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
794 },
795 };
796 bool match_expect = false;
797
798 begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
799 do {
800 match_expect = report_matches(&expect);
801 } while (!end_test_checks(match_expect));
802 KUNIT_EXPECT_TRUE(test, match_expect);
803 }
804
805 __no_kcsan
test_assert_exclusive_bits_nochange(struct kunit * test)806 static void test_assert_exclusive_bits_nochange(struct kunit *test)
807 {
808 bool match_never = false;
809
810 begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
811 do {
812 match_never = report_available();
813 } while (!end_test_checks(match_never));
814 KUNIT_EXPECT_FALSE(test, match_never);
815 }
816
817 __no_kcsan
test_assert_exclusive_writer_scoped(struct kunit * test)818 static void test_assert_exclusive_writer_scoped(struct kunit *test)
819 {
820 const struct expect_report expect_start = {
821 .access = {
822 { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
823 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
824 },
825 };
826 const struct expect_report expect_anywhere = {
827 .access = {
828 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
829 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
830 },
831 };
832 bool match_expect_start = false;
833 bool match_expect_anywhere = false;
834
835 begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
836 do {
837 match_expect_start |= report_matches(&expect_start);
838 match_expect_anywhere |= report_matches(&expect_anywhere);
839 } while (!end_test_checks(match_expect_start && match_expect_anywhere));
840 KUNIT_EXPECT_TRUE(test, match_expect_start);
841 KUNIT_EXPECT_TRUE(test, match_expect_anywhere);
842 }
843
844 __no_kcsan
test_assert_exclusive_access_scoped(struct kunit * test)845 static void test_assert_exclusive_access_scoped(struct kunit *test)
846 {
847 const struct expect_report expect_start1 = {
848 .access = {
849 { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
850 { test_kernel_read, &test_var, sizeof(test_var), 0 },
851 },
852 };
853 const struct expect_report expect_start2 = {
854 .access = { expect_start1.access[0], expect_start1.access[0] },
855 };
856 const struct expect_report expect_inscope = {
857 .access = {
858 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
859 { test_kernel_read, &test_var, sizeof(test_var), 0 },
860 },
861 };
862 bool match_expect_start = false;
863 bool match_expect_inscope = false;
864
865 begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
866 end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
867 do {
868 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
869 match_expect_inscope |= report_matches(&expect_inscope);
870 } while (!end_test_checks(match_expect_start && match_expect_inscope));
871 KUNIT_EXPECT_TRUE(test, match_expect_start);
872 KUNIT_EXPECT_TRUE(test, match_expect_inscope);
873 }
874
875 /*
876 * jiffies is special (declared to be volatile) and its accesses are typically
877 * not marked; this test ensures that the compiler nor KCSAN gets confused about
878 * jiffies's declaration on different architectures.
879 */
880 __no_kcsan
test_jiffies_noreport(struct kunit * test)881 static void test_jiffies_noreport(struct kunit *test)
882 {
883 bool match_never = false;
884
885 begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
886 do {
887 match_never = report_available();
888 } while (!end_test_checks(match_never));
889 KUNIT_EXPECT_FALSE(test, match_never);
890 }
891
892 /* Test that racing accesses in seqlock critical sections are not reported. */
893 __no_kcsan
test_seqlock_noreport(struct kunit * test)894 static void test_seqlock_noreport(struct kunit *test)
895 {
896 bool match_never = false;
897
898 begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
899 do {
900 match_never = report_available();
901 } while (!end_test_checks(match_never));
902 KUNIT_EXPECT_FALSE(test, match_never);
903 }
904
905 /*
906 * Test atomic builtins work and required instrumentation functions exist. We
907 * also test that KCSAN understands they're atomic by racing with them via
908 * test_kernel_atomic_builtins(), and expect no reports.
909 *
910 * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
911 */
test_atomic_builtins(struct kunit * test)912 static void test_atomic_builtins(struct kunit *test)
913 {
914 bool match_never = false;
915
916 begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
917 do {
918 long tmp;
919
920 kcsan_enable_current();
921
922 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
923 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
924
925 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
926 KUNIT_EXPECT_EQ(test, 20L, test_var);
927
928 tmp = 20L;
929 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
930 0, __ATOMIC_RELAXED,
931 __ATOMIC_RELAXED));
932 KUNIT_EXPECT_EQ(test, tmp, 20L);
933 KUNIT_EXPECT_EQ(test, test_var, 30L);
934 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
935 1, __ATOMIC_RELAXED,
936 __ATOMIC_RELAXED));
937 KUNIT_EXPECT_EQ(test, tmp, 30L);
938 KUNIT_EXPECT_EQ(test, test_var, 30L);
939
940 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
941 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
942 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
943 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
944 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
945 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
946 KUNIT_EXPECT_EQ(test, -2L, test_var);
947
948 __atomic_thread_fence(__ATOMIC_SEQ_CST);
949 __atomic_signal_fence(__ATOMIC_SEQ_CST);
950
951 kcsan_disable_current();
952
953 match_never = report_available();
954 } while (!end_test_checks(match_never));
955 KUNIT_EXPECT_FALSE(test, match_never);
956 }
957
958 /*
959 * Each test case is run with different numbers of threads. Until KUnit supports
960 * passing arguments for each test case, we encode #threads in the test case
961 * name (read by get_num_threads()). [The '-' was chosen as a stylistic
962 * preference to separate test name and #threads.]
963 *
964 * The thread counts are chosen to cover potentially interesting boundaries and
965 * corner cases (range 2-5), and then stress the system with larger counts.
966 */
967 #define KCSAN_KUNIT_CASE(test_name) \
968 { .run_case = test_name, .name = #test_name "-02" }, \
969 { .run_case = test_name, .name = #test_name "-03" }, \
970 { .run_case = test_name, .name = #test_name "-04" }, \
971 { .run_case = test_name, .name = #test_name "-05" }, \
972 { .run_case = test_name, .name = #test_name "-08" }, \
973 { .run_case = test_name, .name = #test_name "-16" }
974
975 static struct kunit_case kcsan_test_cases[] = {
976 KCSAN_KUNIT_CASE(test_basic),
977 KCSAN_KUNIT_CASE(test_concurrent_races),
978 KCSAN_KUNIT_CASE(test_novalue_change),
979 KCSAN_KUNIT_CASE(test_novalue_change_exception),
980 KCSAN_KUNIT_CASE(test_unknown_origin),
981 KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
982 KCSAN_KUNIT_CASE(test_write_write_struct),
983 KCSAN_KUNIT_CASE(test_write_write_struct_part),
984 KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
985 KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
986 KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
987 KCSAN_KUNIT_CASE(test_zero_size_access),
988 KCSAN_KUNIT_CASE(test_data_race),
989 KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
990 KCSAN_KUNIT_CASE(test_assert_exclusive_access),
991 KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
992 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
993 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
994 KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
995 KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
996 KCSAN_KUNIT_CASE(test_jiffies_noreport),
997 KCSAN_KUNIT_CASE(test_seqlock_noreport),
998 KCSAN_KUNIT_CASE(test_atomic_builtins),
999 {},
1000 };
1001
1002 /* ===== End test cases ===== */
1003
1004 /* Get number of threads encoded in test name. */
1005 static bool __no_kcsan
get_num_threads(const char * test,int * nthreads)1006 get_num_threads(const char *test, int *nthreads)
1007 {
1008 int len = strlen(test);
1009
1010 if (WARN_ON(len < 3))
1011 return false;
1012
1013 *nthreads = test[len - 1] - '0';
1014 *nthreads += (test[len - 2] - '0') * 10;
1015
1016 if (WARN_ON(*nthreads < 0))
1017 return false;
1018
1019 return true;
1020 }
1021
1022 /* Concurrent accesses from interrupts. */
1023 __no_kcsan
access_thread_timer(struct timer_list * timer)1024 static void access_thread_timer(struct timer_list *timer)
1025 {
1026 static atomic_t cnt = ATOMIC_INIT(0);
1027 unsigned int idx;
1028 void (*func)(void);
1029
1030 idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1031 /* Acquire potential initialization. */
1032 func = smp_load_acquire(&access_kernels[idx]);
1033 if (func)
1034 func();
1035 }
1036
1037 /* The main loop for each thread. */
1038 __no_kcsan
access_thread(void * arg)1039 static int access_thread(void *arg)
1040 {
1041 struct timer_list timer;
1042 unsigned int cnt = 0;
1043 unsigned int idx;
1044 void (*func)(void);
1045
1046 timer_setup_on_stack(&timer, access_thread_timer, 0);
1047 do {
1048 might_sleep();
1049
1050 if (!timer_pending(&timer))
1051 mod_timer(&timer, jiffies + 1);
1052 else {
1053 /* Iterate through all kernels. */
1054 idx = cnt++ % ARRAY_SIZE(access_kernels);
1055 /* Acquire potential initialization. */
1056 func = smp_load_acquire(&access_kernels[idx]);
1057 if (func)
1058 func();
1059 }
1060 } while (!torture_must_stop());
1061 del_timer_sync(&timer);
1062 destroy_timer_on_stack(&timer);
1063
1064 torture_kthread_stopping("access_thread");
1065 return 0;
1066 }
1067
1068 __no_kcsan
test_init(struct kunit * test)1069 static int test_init(struct kunit *test)
1070 {
1071 unsigned long flags;
1072 int nthreads;
1073 int i;
1074
1075 spin_lock_irqsave(&observed.lock, flags);
1076 for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1077 observed.lines[i][0] = '\0';
1078 observed.nlines = 0;
1079 spin_unlock_irqrestore(&observed.lock, flags);
1080
1081 if (!torture_init_begin((char *)test->name, 1))
1082 return -EBUSY;
1083
1084 if (!get_num_threads(test->name, &nthreads))
1085 goto err;
1086
1087 if (WARN_ON(threads))
1088 goto err;
1089
1090 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1091 if (WARN_ON(access_kernels[i]))
1092 goto err;
1093 }
1094
1095 if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1096 /*
1097 * Without any preemption, keep 2 CPUs free for other tasks, one
1098 * of which is the main test case function checking for
1099 * completion or failure.
1100 */
1101 const int min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
1102 const int min_required_cpus = 2 + min_unused_cpus;
1103
1104 if (num_online_cpus() < min_required_cpus) {
1105 pr_err("%s: too few online CPUs (%u < %d) for test",
1106 test->name, num_online_cpus(), min_required_cpus);
1107 goto err;
1108 } else if (nthreads > num_online_cpus() - min_unused_cpus) {
1109 nthreads = num_online_cpus() - min_unused_cpus;
1110 pr_warn("%s: limiting number of threads to %d\n",
1111 test->name, nthreads);
1112 }
1113 }
1114
1115 if (nthreads) {
1116 threads = kcalloc(nthreads + 1, sizeof(struct task_struct *),
1117 GFP_KERNEL);
1118 if (WARN_ON(!threads))
1119 goto err;
1120
1121 threads[nthreads] = NULL;
1122 for (i = 0; i < nthreads; ++i) {
1123 if (torture_create_kthread(access_thread, NULL,
1124 threads[i]))
1125 goto err;
1126 }
1127 }
1128
1129 torture_init_end();
1130
1131 return 0;
1132
1133 err:
1134 kfree(threads);
1135 threads = NULL;
1136 torture_init_end();
1137 return -EINVAL;
1138 }
1139
1140 __no_kcsan
test_exit(struct kunit * test)1141 static void test_exit(struct kunit *test)
1142 {
1143 struct task_struct **stop_thread;
1144 int i;
1145
1146 if (torture_cleanup_begin())
1147 return;
1148
1149 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1150 WRITE_ONCE(access_kernels[i], NULL);
1151
1152 if (threads) {
1153 for (stop_thread = threads; *stop_thread; stop_thread++)
1154 torture_stop_kthread(reader_thread, *stop_thread);
1155
1156 kfree(threads);
1157 threads = NULL;
1158 }
1159
1160 torture_cleanup_end();
1161 }
1162
1163 static struct kunit_suite kcsan_test_suite = {
1164 .name = "kcsan-test",
1165 .test_cases = kcsan_test_cases,
1166 .init = test_init,
1167 .exit = test_exit,
1168 };
1169 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL };
1170
1171 __no_kcsan
register_tracepoints(struct tracepoint * tp,void * ignore)1172 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1173 {
1174 check_trace_callback_type_console(probe_console);
1175 if (!strcmp(tp->name, "console"))
1176 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1177 }
1178
1179 __no_kcsan
unregister_tracepoints(struct tracepoint * tp,void * ignore)1180 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1181 {
1182 if (!strcmp(tp->name, "console"))
1183 tracepoint_probe_unregister(tp, probe_console, NULL);
1184 }
1185
1186 /*
1187 * We only want to do tracepoints setup and teardown once, therefore we have to
1188 * customize the init and exit functions and cannot rely on kunit_test_suite().
1189 */
kcsan_test_init(void)1190 static int __init kcsan_test_init(void)
1191 {
1192 /*
1193 * Because we want to be able to build the test as a module, we need to
1194 * iterate through all known tracepoints, since the static registration
1195 * won't work here.
1196 */
1197 for_each_kernel_tracepoint(register_tracepoints, NULL);
1198 return __kunit_test_suites_init(kcsan_test_suites);
1199 }
1200
kcsan_test_exit(void)1201 static void kcsan_test_exit(void)
1202 {
1203 __kunit_test_suites_exit(kcsan_test_suites);
1204 for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1205 tracepoint_synchronize_unregister();
1206 }
1207
1208 late_initcall(kcsan_test_init);
1209 module_exit(kcsan_test_exit);
1210
1211 MODULE_LICENSE("GPL v2");
1212 MODULE_AUTHOR("Marco Elver <elver@google.com>");
1213