1 // SPDX-License-Identifier: MIT
2
3 /*
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <linux/delay.h>
8 #include <linux/dma-fence.h>
9 #include <linux/dma-fence-chain.h>
10 #include <linux/kernel.h>
11 #include <linux/kthread.h>
12 #include <linux/mm.h>
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 #include <linux/random.h>
17
18 #include "selftest.h"
19
20 #define CHAIN_SZ (4 << 10)
21
22 static struct kmem_cache *slab_fences;
23
24 static inline struct mock_fence {
25 struct dma_fence base;
26 spinlock_t lock;
to_mock_fence(struct dma_fence * f)27 } *to_mock_fence(struct dma_fence *f) {
28 return container_of(f, struct mock_fence, base);
29 }
30
mock_name(struct dma_fence * f)31 static const char *mock_name(struct dma_fence *f)
32 {
33 return "mock";
34 }
35
mock_fence_release(struct dma_fence * f)36 static void mock_fence_release(struct dma_fence *f)
37 {
38 kmem_cache_free(slab_fences, to_mock_fence(f));
39 }
40
41 static const struct dma_fence_ops mock_ops = {
42 .get_driver_name = mock_name,
43 .get_timeline_name = mock_name,
44 .release = mock_fence_release,
45 };
46
mock_fence(void)47 static struct dma_fence *mock_fence(void)
48 {
49 struct mock_fence *f;
50
51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
52 if (!f)
53 return NULL;
54
55 spin_lock_init(&f->lock);
56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
57
58 return &f->base;
59 }
60
mock_chain(struct dma_fence * prev,struct dma_fence * fence,u64 seqno)61 static struct dma_fence *mock_chain(struct dma_fence *prev,
62 struct dma_fence *fence,
63 u64 seqno)
64 {
65 struct dma_fence_chain *f;
66
67 f = dma_fence_chain_alloc();
68 if (!f)
69 return NULL;
70
71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence),
72 seqno);
73
74 return &f->base;
75 }
76
sanitycheck(void * arg)77 static int sanitycheck(void *arg)
78 {
79 struct dma_fence *f, *chain;
80 int err = 0;
81
82 f = mock_fence();
83 if (!f)
84 return -ENOMEM;
85
86 chain = mock_chain(NULL, f, 1);
87 if (!chain)
88 err = -ENOMEM;
89
90 dma_fence_enable_sw_signaling(chain);
91
92 dma_fence_signal(f);
93 dma_fence_put(f);
94
95 dma_fence_put(chain);
96
97 return err;
98 }
99
100 struct fence_chains {
101 unsigned int chain_length;
102 struct dma_fence **fences;
103 struct dma_fence **chains;
104
105 struct dma_fence *tail;
106 };
107
seqno_inc(unsigned int i)108 static uint64_t seqno_inc(unsigned int i)
109 {
110 return i + 1;
111 }
112
fence_chains_init(struct fence_chains * fc,unsigned int count,uint64_t (* seqno_fn)(unsigned int))113 static int fence_chains_init(struct fence_chains *fc, unsigned int count,
114 uint64_t (*seqno_fn)(unsigned int))
115 {
116 unsigned int i;
117 int err = 0;
118
119 fc->chains = kvmalloc_array(count, sizeof(*fc->chains),
120 GFP_KERNEL | __GFP_ZERO);
121 if (!fc->chains)
122 return -ENOMEM;
123
124 fc->fences = kvmalloc_array(count, sizeof(*fc->fences),
125 GFP_KERNEL | __GFP_ZERO);
126 if (!fc->fences) {
127 err = -ENOMEM;
128 goto err_chains;
129 }
130
131 fc->tail = NULL;
132 for (i = 0; i < count; i++) {
133 fc->fences[i] = mock_fence();
134 if (!fc->fences[i]) {
135 err = -ENOMEM;
136 goto unwind;
137 }
138
139 fc->chains[i] = mock_chain(fc->tail,
140 fc->fences[i],
141 seqno_fn(i));
142 if (!fc->chains[i]) {
143 err = -ENOMEM;
144 goto unwind;
145 }
146
147 fc->tail = fc->chains[i];
148
149 dma_fence_enable_sw_signaling(fc->chains[i]);
150 }
151
152 fc->chain_length = i;
153 return 0;
154
155 unwind:
156 for (i = 0; i < count; i++) {
157 dma_fence_put(fc->fences[i]);
158 dma_fence_put(fc->chains[i]);
159 }
160 kvfree(fc->fences);
161 err_chains:
162 kvfree(fc->chains);
163 return err;
164 }
165
fence_chains_fini(struct fence_chains * fc)166 static void fence_chains_fini(struct fence_chains *fc)
167 {
168 unsigned int i;
169
170 for (i = 0; i < fc->chain_length; i++) {
171 dma_fence_signal(fc->fences[i]);
172 dma_fence_put(fc->fences[i]);
173 }
174 kvfree(fc->fences);
175
176 for (i = 0; i < fc->chain_length; i++)
177 dma_fence_put(fc->chains[i]);
178 kvfree(fc->chains);
179 }
180
find_seqno(void * arg)181 static int find_seqno(void *arg)
182 {
183 struct fence_chains fc;
184 struct dma_fence *fence;
185 int err;
186 int i;
187
188 err = fence_chains_init(&fc, 64, seqno_inc);
189 if (err)
190 return err;
191
192 fence = dma_fence_get(fc.tail);
193 err = dma_fence_chain_find_seqno(&fence, 0);
194 dma_fence_put(fence);
195 if (err) {
196 pr_err("Reported %d for find_seqno(0)!\n", err);
197 goto err;
198 }
199
200 for (i = 0; i < fc.chain_length; i++) {
201 fence = dma_fence_get(fc.tail);
202 err = dma_fence_chain_find_seqno(&fence, i + 1);
203 dma_fence_put(fence);
204 if (err) {
205 pr_err("Reported %d for find_seqno(%d:%d)!\n",
206 err, fc.chain_length + 1, i + 1);
207 goto err;
208 }
209 if (fence != fc.chains[i]) {
210 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
211 fc.chain_length + 1, i + 1);
212 err = -EINVAL;
213 goto err;
214 }
215
216 dma_fence_get(fence);
217 err = dma_fence_chain_find_seqno(&fence, i + 1);
218 dma_fence_put(fence);
219 if (err) {
220 pr_err("Error reported for finding self\n");
221 goto err;
222 }
223 if (fence != fc.chains[i]) {
224 pr_err("Incorrect fence reported by find self\n");
225 err = -EINVAL;
226 goto err;
227 }
228
229 dma_fence_get(fence);
230 err = dma_fence_chain_find_seqno(&fence, i + 2);
231 dma_fence_put(fence);
232 if (!err) {
233 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
234 i + 1, i + 2);
235 err = -EINVAL;
236 goto err;
237 }
238
239 dma_fence_get(fence);
240 err = dma_fence_chain_find_seqno(&fence, i);
241 dma_fence_put(fence);
242 if (err) {
243 pr_err("Error reported for previous fence!\n");
244 goto err;
245 }
246 if (i > 0 && fence != fc.chains[i - 1]) {
247 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
248 i + 1, i);
249 err = -EINVAL;
250 goto err;
251 }
252 }
253
254 err:
255 fence_chains_fini(&fc);
256 return err;
257 }
258
find_signaled(void * arg)259 static int find_signaled(void *arg)
260 {
261 struct fence_chains fc;
262 struct dma_fence *fence;
263 int err;
264
265 err = fence_chains_init(&fc, 2, seqno_inc);
266 if (err)
267 return err;
268
269 dma_fence_signal(fc.fences[0]);
270
271 fence = dma_fence_get(fc.tail);
272 err = dma_fence_chain_find_seqno(&fence, 1);
273 dma_fence_put(fence);
274 if (err) {
275 pr_err("Reported %d for find_seqno()!\n", err);
276 goto err;
277 }
278
279 if (fence && fence != fc.chains[0]) {
280 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
281 fence->seqno);
282
283 dma_fence_get(fence);
284 err = dma_fence_chain_find_seqno(&fence, 1);
285 dma_fence_put(fence);
286 if (err)
287 pr_err("Reported %d for finding self!\n", err);
288
289 err = -EINVAL;
290 }
291
292 err:
293 fence_chains_fini(&fc);
294 return err;
295 }
296
find_out_of_order(void * arg)297 static int find_out_of_order(void *arg)
298 {
299 struct fence_chains fc;
300 struct dma_fence *fence;
301 int err;
302
303 err = fence_chains_init(&fc, 3, seqno_inc);
304 if (err)
305 return err;
306
307 dma_fence_signal(fc.fences[1]);
308
309 fence = dma_fence_get(fc.tail);
310 err = dma_fence_chain_find_seqno(&fence, 2);
311 dma_fence_put(fence);
312 if (err) {
313 pr_err("Reported %d for find_seqno()!\n", err);
314 goto err;
315 }
316
317 /*
318 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior
319 * of the dma-fence-chain is to make us wait for all the fences up to
320 * the point we want. Since fence 1 is still not signaled, this what
321 * we should get as fence to wait upon (fence 2 being garbage
322 * collected during the traversal of the chain).
323 */
324 if (fence != fc.chains[0]) {
325 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
326 fence ? fence->seqno : 0);
327
328 err = -EINVAL;
329 }
330
331 err:
332 fence_chains_fini(&fc);
333 return err;
334 }
335
seqno_inc2(unsigned int i)336 static uint64_t seqno_inc2(unsigned int i)
337 {
338 return 2 * i + 2;
339 }
340
find_gap(void * arg)341 static int find_gap(void *arg)
342 {
343 struct fence_chains fc;
344 struct dma_fence *fence;
345 int err;
346 int i;
347
348 err = fence_chains_init(&fc, 64, seqno_inc2);
349 if (err)
350 return err;
351
352 for (i = 0; i < fc.chain_length; i++) {
353 fence = dma_fence_get(fc.tail);
354 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
355 dma_fence_put(fence);
356 if (err) {
357 pr_err("Reported %d for find_seqno(%d:%d)!\n",
358 err, fc.chain_length + 1, 2 * i + 1);
359 goto err;
360 }
361 if (fence != fc.chains[i]) {
362 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
363 fence->seqno,
364 fc.chain_length + 1,
365 2 * i + 1);
366 err = -EINVAL;
367 goto err;
368 }
369
370 dma_fence_get(fence);
371 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
372 dma_fence_put(fence);
373 if (err) {
374 pr_err("Error reported for finding self\n");
375 goto err;
376 }
377 if (fence != fc.chains[i]) {
378 pr_err("Incorrect fence reported by find self\n");
379 err = -EINVAL;
380 goto err;
381 }
382 }
383
384 err:
385 fence_chains_fini(&fc);
386 return err;
387 }
388
389 struct find_race {
390 struct fence_chains fc;
391 atomic_t children;
392 };
393
__find_race(void * arg)394 static int __find_race(void *arg)
395 {
396 struct find_race *data = arg;
397 int err = 0;
398
399 while (!kthread_should_stop()) {
400 struct dma_fence *fence = dma_fence_get(data->fc.tail);
401 int seqno;
402
403 seqno = prandom_u32_max(data->fc.chain_length) + 1;
404
405 err = dma_fence_chain_find_seqno(&fence, seqno);
406 if (err) {
407 pr_err("Failed to find fence seqno:%d\n",
408 seqno);
409 dma_fence_put(fence);
410 break;
411 }
412 if (!fence)
413 goto signal;
414
415 /*
416 * We can only find ourselves if we are on fence we were
417 * looking for.
418 */
419 if (fence->seqno == seqno) {
420 err = dma_fence_chain_find_seqno(&fence, seqno);
421 if (err) {
422 pr_err("Reported an invalid fence for find-self:%d\n",
423 seqno);
424 dma_fence_put(fence);
425 break;
426 }
427 }
428
429 dma_fence_put(fence);
430
431 signal:
432 seqno = prandom_u32_max(data->fc.chain_length - 1);
433 dma_fence_signal(data->fc.fences[seqno]);
434 cond_resched();
435 }
436
437 if (atomic_dec_and_test(&data->children))
438 wake_up_var(&data->children);
439 return err;
440 }
441
find_race(void * arg)442 static int find_race(void *arg)
443 {
444 struct find_race data;
445 int ncpus = num_online_cpus();
446 struct task_struct **threads;
447 unsigned long count;
448 int err;
449 int i;
450
451 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
452 if (err)
453 return err;
454
455 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);
456 if (!threads) {
457 err = -ENOMEM;
458 goto err;
459 }
460
461 atomic_set(&data.children, 0);
462 for (i = 0; i < ncpus; i++) {
463 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
464 if (IS_ERR(threads[i])) {
465 ncpus = i;
466 break;
467 }
468 atomic_inc(&data.children);
469 get_task_struct(threads[i]);
470 }
471
472 wait_var_event_timeout(&data.children,
473 !atomic_read(&data.children),
474 5 * HZ);
475
476 for (i = 0; i < ncpus; i++) {
477 int ret;
478
479 ret = kthread_stop(threads[i]);
480 if (ret && !err)
481 err = ret;
482 put_task_struct(threads[i]);
483 }
484 kfree(threads);
485
486 count = 0;
487 for (i = 0; i < data.fc.chain_length; i++)
488 if (dma_fence_is_signaled(data.fc.fences[i]))
489 count++;
490 pr_info("Completed %lu cycles\n", count);
491
492 err:
493 fence_chains_fini(&data.fc);
494 return err;
495 }
496
signal_forward(void * arg)497 static int signal_forward(void *arg)
498 {
499 struct fence_chains fc;
500 int err;
501 int i;
502
503 err = fence_chains_init(&fc, 64, seqno_inc);
504 if (err)
505 return err;
506
507 for (i = 0; i < fc.chain_length; i++) {
508 dma_fence_signal(fc.fences[i]);
509
510 if (!dma_fence_is_signaled(fc.chains[i])) {
511 pr_err("chain[%d] not signaled!\n", i);
512 err = -EINVAL;
513 goto err;
514 }
515
516 if (i + 1 < fc.chain_length &&
517 dma_fence_is_signaled(fc.chains[i + 1])) {
518 pr_err("chain[%d] is signaled!\n", i);
519 err = -EINVAL;
520 goto err;
521 }
522 }
523
524 err:
525 fence_chains_fini(&fc);
526 return err;
527 }
528
signal_backward(void * arg)529 static int signal_backward(void *arg)
530 {
531 struct fence_chains fc;
532 int err;
533 int i;
534
535 err = fence_chains_init(&fc, 64, seqno_inc);
536 if (err)
537 return err;
538
539 for (i = fc.chain_length; i--; ) {
540 dma_fence_signal(fc.fences[i]);
541
542 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
543 pr_err("chain[%d] is signaled!\n", i);
544 err = -EINVAL;
545 goto err;
546 }
547 }
548
549 for (i = 0; i < fc.chain_length; i++) {
550 if (!dma_fence_is_signaled(fc.chains[i])) {
551 pr_err("chain[%d] was not signaled!\n", i);
552 err = -EINVAL;
553 goto err;
554 }
555 }
556
557 err:
558 fence_chains_fini(&fc);
559 return err;
560 }
561
__wait_fence_chains(void * arg)562 static int __wait_fence_chains(void *arg)
563 {
564 struct fence_chains *fc = arg;
565
566 if (dma_fence_wait(fc->tail, false))
567 return -EIO;
568
569 return 0;
570 }
571
wait_forward(void * arg)572 static int wait_forward(void *arg)
573 {
574 struct fence_chains fc;
575 struct task_struct *tsk;
576 int err;
577 int i;
578
579 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
580 if (err)
581 return err;
582
583 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
584 if (IS_ERR(tsk)) {
585 err = PTR_ERR(tsk);
586 goto err;
587 }
588 get_task_struct(tsk);
589 yield_to(tsk, true);
590
591 for (i = 0; i < fc.chain_length; i++)
592 dma_fence_signal(fc.fences[i]);
593
594 err = kthread_stop(tsk);
595 put_task_struct(tsk);
596
597 err:
598 fence_chains_fini(&fc);
599 return err;
600 }
601
wait_backward(void * arg)602 static int wait_backward(void *arg)
603 {
604 struct fence_chains fc;
605 struct task_struct *tsk;
606 int err;
607 int i;
608
609 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
610 if (err)
611 return err;
612
613 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
614 if (IS_ERR(tsk)) {
615 err = PTR_ERR(tsk);
616 goto err;
617 }
618 get_task_struct(tsk);
619 yield_to(tsk, true);
620
621 for (i = fc.chain_length; i--; )
622 dma_fence_signal(fc.fences[i]);
623
624 err = kthread_stop(tsk);
625 put_task_struct(tsk);
626
627 err:
628 fence_chains_fini(&fc);
629 return err;
630 }
631
randomise_fences(struct fence_chains * fc)632 static void randomise_fences(struct fence_chains *fc)
633 {
634 unsigned int count = fc->chain_length;
635
636 /* Fisher-Yates shuffle courtesy of Knuth */
637 while (--count) {
638 unsigned int swp;
639
640 swp = prandom_u32_max(count + 1);
641 if (swp == count)
642 continue;
643
644 swap(fc->fences[count], fc->fences[swp]);
645 }
646 }
647
wait_random(void * arg)648 static int wait_random(void *arg)
649 {
650 struct fence_chains fc;
651 struct task_struct *tsk;
652 int err;
653 int i;
654
655 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
656 if (err)
657 return err;
658
659 randomise_fences(&fc);
660
661 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
662 if (IS_ERR(tsk)) {
663 err = PTR_ERR(tsk);
664 goto err;
665 }
666 get_task_struct(tsk);
667 yield_to(tsk, true);
668
669 for (i = 0; i < fc.chain_length; i++)
670 dma_fence_signal(fc.fences[i]);
671
672 err = kthread_stop(tsk);
673 put_task_struct(tsk);
674
675 err:
676 fence_chains_fini(&fc);
677 return err;
678 }
679
dma_fence_chain(void)680 int dma_fence_chain(void)
681 {
682 static const struct subtest tests[] = {
683 SUBTEST(sanitycheck),
684 SUBTEST(find_seqno),
685 SUBTEST(find_signaled),
686 SUBTEST(find_out_of_order),
687 SUBTEST(find_gap),
688 SUBTEST(find_race),
689 SUBTEST(signal_forward),
690 SUBTEST(signal_backward),
691 SUBTEST(wait_forward),
692 SUBTEST(wait_backward),
693 SUBTEST(wait_random),
694 };
695 int ret;
696
697 pr_info("sizeof(dma_fence_chain)=%zu\n",
698 sizeof(struct dma_fence_chain));
699
700 slab_fences = KMEM_CACHE(mock_fence,
701 SLAB_TYPESAFE_BY_RCU |
702 SLAB_HWCACHE_ALIGN);
703 if (!slab_fences)
704 return -ENOMEM;
705
706 ret = subtests(tests, NULL);
707
708 kmem_cache_destroy(slab_fences);
709 return ret;
710 }
711