1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2016 Intel Corporation
4 */
5
6 #include <linux/kthread.h>
7
8 #include "gem/i915_gem_context.h"
9
10 #include "intel_gt.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_engine_pm.h"
13 #include "selftest_engine_heartbeat.h"
14
15 #include "i915_selftest.h"
16 #include "selftests/i915_random.h"
17 #include "selftests/igt_flush_test.h"
18 #include "selftests/igt_reset.h"
19 #include "selftests/igt_atomic.h"
20 #include "selftests/igt_spinner.h"
21 #include "selftests/intel_scheduler_helpers.h"
22
23 #include "selftests/mock_drm.h"
24
25 #include "gem/selftests/mock_context.h"
26 #include "gem/selftests/igt_gem_utils.h"
27
28 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
29
30 struct hang {
31 struct intel_gt *gt;
32 struct drm_i915_gem_object *hws;
33 struct drm_i915_gem_object *obj;
34 struct i915_gem_context *ctx;
35 u32 *seqno;
36 u32 *batch;
37 };
38
hang_init(struct hang * h,struct intel_gt * gt)39 static int hang_init(struct hang *h, struct intel_gt *gt)
40 {
41 void *vaddr;
42 int err;
43
44 memset(h, 0, sizeof(*h));
45 h->gt = gt;
46
47 h->ctx = kernel_context(gt->i915, NULL);
48 if (IS_ERR(h->ctx))
49 return PTR_ERR(h->ctx);
50
51 GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));
52
53 h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
54 if (IS_ERR(h->hws)) {
55 err = PTR_ERR(h->hws);
56 goto err_ctx;
57 }
58
59 h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
60 if (IS_ERR(h->obj)) {
61 err = PTR_ERR(h->obj);
62 goto err_hws;
63 }
64
65 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
66 vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
67 if (IS_ERR(vaddr)) {
68 err = PTR_ERR(vaddr);
69 goto err_obj;
70 }
71 h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
72
73 vaddr = i915_gem_object_pin_map_unlocked(h->obj,
74 i915_coherent_map_type(gt->i915, h->obj, false));
75 if (IS_ERR(vaddr)) {
76 err = PTR_ERR(vaddr);
77 goto err_unpin_hws;
78 }
79 h->batch = vaddr;
80
81 return 0;
82
83 err_unpin_hws:
84 i915_gem_object_unpin_map(h->hws);
85 err_obj:
86 i915_gem_object_put(h->obj);
87 err_hws:
88 i915_gem_object_put(h->hws);
89 err_ctx:
90 kernel_context_close(h->ctx);
91 return err;
92 }
93
hws_address(const struct i915_vma * hws,const struct i915_request * rq)94 static u64 hws_address(const struct i915_vma *hws,
95 const struct i915_request *rq)
96 {
97 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
98 }
99
move_to_active(struct i915_vma * vma,struct i915_request * rq,unsigned int flags)100 static int move_to_active(struct i915_vma *vma,
101 struct i915_request *rq,
102 unsigned int flags)
103 {
104 int err;
105
106 i915_vma_lock(vma);
107 err = i915_request_await_object(rq, vma->obj,
108 flags & EXEC_OBJECT_WRITE);
109 if (err == 0)
110 err = i915_vma_move_to_active(vma, rq, flags);
111 i915_vma_unlock(vma);
112
113 return err;
114 }
115
116 static struct i915_request *
hang_create_request(struct hang * h,struct intel_engine_cs * engine)117 hang_create_request(struct hang *h, struct intel_engine_cs *engine)
118 {
119 struct intel_gt *gt = h->gt;
120 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(h->ctx);
121 struct drm_i915_gem_object *obj;
122 struct i915_request *rq = NULL;
123 struct i915_vma *hws, *vma;
124 unsigned int flags;
125 void *vaddr;
126 u32 *batch;
127 int err;
128
129 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
130 if (IS_ERR(obj)) {
131 i915_vm_put(vm);
132 return ERR_CAST(obj);
133 }
134
135 vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false));
136 if (IS_ERR(vaddr)) {
137 i915_gem_object_put(obj);
138 i915_vm_put(vm);
139 return ERR_CAST(vaddr);
140 }
141
142 i915_gem_object_unpin_map(h->obj);
143 i915_gem_object_put(h->obj);
144
145 h->obj = obj;
146 h->batch = vaddr;
147
148 vma = i915_vma_instance(h->obj, vm, NULL);
149 if (IS_ERR(vma)) {
150 i915_vm_put(vm);
151 return ERR_CAST(vma);
152 }
153
154 hws = i915_vma_instance(h->hws, vm, NULL);
155 if (IS_ERR(hws)) {
156 i915_vm_put(vm);
157 return ERR_CAST(hws);
158 }
159
160 err = i915_vma_pin(vma, 0, 0, PIN_USER);
161 if (err) {
162 i915_vm_put(vm);
163 return ERR_PTR(err);
164 }
165
166 err = i915_vma_pin(hws, 0, 0, PIN_USER);
167 if (err)
168 goto unpin_vma;
169
170 rq = igt_request_alloc(h->ctx, engine);
171 if (IS_ERR(rq)) {
172 err = PTR_ERR(rq);
173 goto unpin_hws;
174 }
175
176 err = move_to_active(vma, rq, 0);
177 if (err)
178 goto cancel_rq;
179
180 err = move_to_active(hws, rq, 0);
181 if (err)
182 goto cancel_rq;
183
184 batch = h->batch;
185 if (GRAPHICS_VER(gt->i915) >= 8) {
186 *batch++ = MI_STORE_DWORD_IMM_GEN4;
187 *batch++ = lower_32_bits(hws_address(hws, rq));
188 *batch++ = upper_32_bits(hws_address(hws, rq));
189 *batch++ = rq->fence.seqno;
190 *batch++ = MI_NOOP;
191
192 memset(batch, 0, 1024);
193 batch += 1024 / sizeof(*batch);
194
195 *batch++ = MI_NOOP;
196 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
197 *batch++ = lower_32_bits(vma->node.start);
198 *batch++ = upper_32_bits(vma->node.start);
199 } else if (GRAPHICS_VER(gt->i915) >= 6) {
200 *batch++ = MI_STORE_DWORD_IMM_GEN4;
201 *batch++ = 0;
202 *batch++ = lower_32_bits(hws_address(hws, rq));
203 *batch++ = rq->fence.seqno;
204 *batch++ = MI_NOOP;
205
206 memset(batch, 0, 1024);
207 batch += 1024 / sizeof(*batch);
208
209 *batch++ = MI_NOOP;
210 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;
211 *batch++ = lower_32_bits(vma->node.start);
212 } else if (GRAPHICS_VER(gt->i915) >= 4) {
213 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
214 *batch++ = 0;
215 *batch++ = lower_32_bits(hws_address(hws, rq));
216 *batch++ = rq->fence.seqno;
217 *batch++ = MI_NOOP;
218
219 memset(batch, 0, 1024);
220 batch += 1024 / sizeof(*batch);
221
222 *batch++ = MI_NOOP;
223 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
224 *batch++ = lower_32_bits(vma->node.start);
225 } else {
226 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
227 *batch++ = lower_32_bits(hws_address(hws, rq));
228 *batch++ = rq->fence.seqno;
229 *batch++ = MI_NOOP;
230
231 memset(batch, 0, 1024);
232 batch += 1024 / sizeof(*batch);
233
234 *batch++ = MI_NOOP;
235 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
236 *batch++ = lower_32_bits(vma->node.start);
237 }
238 *batch++ = MI_BATCH_BUFFER_END; /* not reached */
239 intel_gt_chipset_flush(engine->gt);
240
241 if (rq->engine->emit_init_breadcrumb) {
242 err = rq->engine->emit_init_breadcrumb(rq);
243 if (err)
244 goto cancel_rq;
245 }
246
247 flags = 0;
248 if (GRAPHICS_VER(gt->i915) <= 5)
249 flags |= I915_DISPATCH_SECURE;
250
251 err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
252
253 cancel_rq:
254 if (err) {
255 i915_request_set_error_once(rq, err);
256 i915_request_add(rq);
257 }
258 unpin_hws:
259 i915_vma_unpin(hws);
260 unpin_vma:
261 i915_vma_unpin(vma);
262 i915_vm_put(vm);
263 return err ? ERR_PTR(err) : rq;
264 }
265
hws_seqno(const struct hang * h,const struct i915_request * rq)266 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
267 {
268 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
269 }
270
hang_fini(struct hang * h)271 static void hang_fini(struct hang *h)
272 {
273 *h->batch = MI_BATCH_BUFFER_END;
274 intel_gt_chipset_flush(h->gt);
275
276 i915_gem_object_unpin_map(h->obj);
277 i915_gem_object_put(h->obj);
278
279 i915_gem_object_unpin_map(h->hws);
280 i915_gem_object_put(h->hws);
281
282 kernel_context_close(h->ctx);
283
284 igt_flush_test(h->gt->i915);
285 }
286
wait_until_running(struct hang * h,struct i915_request * rq)287 static bool wait_until_running(struct hang *h, struct i915_request *rq)
288 {
289 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
290 rq->fence.seqno),
291 10) &&
292 wait_for(i915_seqno_passed(hws_seqno(h, rq),
293 rq->fence.seqno),
294 1000));
295 }
296
igt_hang_sanitycheck(void * arg)297 static int igt_hang_sanitycheck(void *arg)
298 {
299 struct intel_gt *gt = arg;
300 struct i915_request *rq;
301 struct intel_engine_cs *engine;
302 enum intel_engine_id id;
303 struct hang h;
304 int err;
305
306 /* Basic check that we can execute our hanging batch */
307
308 err = hang_init(&h, gt);
309 if (err)
310 return err;
311
312 for_each_engine(engine, gt, id) {
313 struct intel_wedge_me w;
314 long timeout;
315
316 if (!intel_engine_can_store_dword(engine))
317 continue;
318
319 rq = hang_create_request(&h, engine);
320 if (IS_ERR(rq)) {
321 err = PTR_ERR(rq);
322 pr_err("Failed to create request for %s, err=%d\n",
323 engine->name, err);
324 goto fini;
325 }
326
327 i915_request_get(rq);
328
329 *h.batch = MI_BATCH_BUFFER_END;
330 intel_gt_chipset_flush(engine->gt);
331
332 i915_request_add(rq);
333
334 timeout = 0;
335 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
336 timeout = i915_request_wait(rq, 0,
337 MAX_SCHEDULE_TIMEOUT);
338 if (intel_gt_is_wedged(gt))
339 timeout = -EIO;
340
341 i915_request_put(rq);
342
343 if (timeout < 0) {
344 err = timeout;
345 pr_err("Wait for request failed on %s, err=%d\n",
346 engine->name, err);
347 goto fini;
348 }
349 }
350
351 fini:
352 hang_fini(&h);
353 return err;
354 }
355
wait_for_idle(struct intel_engine_cs * engine)356 static bool wait_for_idle(struct intel_engine_cs *engine)
357 {
358 return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
359 }
360
igt_reset_nop(void * arg)361 static int igt_reset_nop(void *arg)
362 {
363 struct intel_gt *gt = arg;
364 struct i915_gpu_error *global = >->i915->gpu_error;
365 struct intel_engine_cs *engine;
366 unsigned int reset_count, count;
367 enum intel_engine_id id;
368 IGT_TIMEOUT(end_time);
369 int err = 0;
370
371 /* Check that we can reset during non-user portions of requests */
372
373 reset_count = i915_reset_count(global);
374 count = 0;
375 do {
376 for_each_engine(engine, gt, id) {
377 struct intel_context *ce;
378 int i;
379
380 ce = intel_context_create(engine);
381 if (IS_ERR(ce)) {
382 err = PTR_ERR(ce);
383 pr_err("[%s] Create context failed: %d!\n", engine->name, err);
384 break;
385 }
386
387 for (i = 0; i < 16; i++) {
388 struct i915_request *rq;
389
390 rq = intel_context_create_request(ce);
391 if (IS_ERR(rq)) {
392 err = PTR_ERR(rq);
393 pr_err("[%s] Create request failed: %d!\n",
394 engine->name, err);
395 break;
396 }
397
398 i915_request_add(rq);
399 }
400
401 intel_context_put(ce);
402 }
403
404 igt_global_reset_lock(gt);
405 intel_gt_reset(gt, ALL_ENGINES, NULL);
406 igt_global_reset_unlock(gt);
407
408 if (intel_gt_is_wedged(gt)) {
409 pr_err("[%s] GT is wedged!\n", engine->name);
410 err = -EIO;
411 break;
412 }
413
414 if (i915_reset_count(global) != reset_count + ++count) {
415 pr_err("[%s] Reset not recorded: %d vs %d + %d!\n",
416 engine->name, i915_reset_count(global), reset_count, count);
417 err = -EINVAL;
418 break;
419 }
420
421 err = igt_flush_test(gt->i915);
422 if (err) {
423 pr_err("[%s] Flush failed: %d!\n", engine->name, err);
424 break;
425 }
426 } while (time_before(jiffies, end_time));
427 pr_info("%s: %d resets\n", __func__, count);
428
429 if (igt_flush_test(gt->i915)) {
430 pr_err("Post flush failed: %d!\n", err);
431 err = -EIO;
432 }
433
434 return err;
435 }
436
igt_reset_nop_engine(void * arg)437 static int igt_reset_nop_engine(void *arg)
438 {
439 struct intel_gt *gt = arg;
440 struct i915_gpu_error *global = >->i915->gpu_error;
441 struct intel_engine_cs *engine;
442 enum intel_engine_id id;
443
444 /* Check that we can engine-reset during non-user portions */
445
446 if (!intel_has_reset_engine(gt))
447 return 0;
448
449 for_each_engine(engine, gt, id) {
450 unsigned int reset_count, reset_engine_count, count;
451 struct intel_context *ce;
452 IGT_TIMEOUT(end_time);
453 int err;
454
455 if (intel_engine_uses_guc(engine)) {
456 /* Engine level resets are triggered by GuC when a hang
457 * is detected. They can't be triggered by the KMD any
458 * more. Thus a nop batch cannot be used as a reset test
459 */
460 continue;
461 }
462
463 ce = intel_context_create(engine);
464 if (IS_ERR(ce)) {
465 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);
466 return PTR_ERR(ce);
467 }
468
469 reset_count = i915_reset_count(global);
470 reset_engine_count = i915_reset_engine_count(global, engine);
471 count = 0;
472
473 st_engine_heartbeat_disable(engine);
474 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
475 do {
476 int i;
477
478 if (!wait_for_idle(engine)) {
479 pr_err("%s failed to idle before reset\n",
480 engine->name);
481 err = -EIO;
482 break;
483 }
484
485 for (i = 0; i < 16; i++) {
486 struct i915_request *rq;
487
488 rq = intel_context_create_request(ce);
489 if (IS_ERR(rq)) {
490 struct drm_printer p =
491 drm_info_printer(gt->i915->drm.dev);
492 intel_engine_dump(engine, &p,
493 "%s(%s): failed to submit request\n",
494 __func__,
495 engine->name);
496
497 GEM_TRACE("%s(%s): failed to submit request\n",
498 __func__,
499 engine->name);
500 GEM_TRACE_DUMP();
501
502 intel_gt_set_wedged(gt);
503
504 err = PTR_ERR(rq);
505 break;
506 }
507
508 i915_request_add(rq);
509 }
510 err = intel_engine_reset(engine, NULL);
511 if (err) {
512 pr_err("intel_engine_reset(%s) failed, err:%d\n",
513 engine->name, err);
514 break;
515 }
516
517 if (i915_reset_count(global) != reset_count) {
518 pr_err("Full GPU reset recorded! (engine reset expected)\n");
519 err = -EINVAL;
520 break;
521 }
522
523 if (i915_reset_engine_count(global, engine) !=
524 reset_engine_count + ++count) {
525 pr_err("%s engine reset not recorded!\n",
526 engine->name);
527 err = -EINVAL;
528 break;
529 }
530 } while (time_before(jiffies, end_time));
531 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
532 st_engine_heartbeat_enable(engine);
533
534 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
535
536 intel_context_put(ce);
537 if (igt_flush_test(gt->i915))
538 err = -EIO;
539 if (err)
540 return err;
541 }
542
543 return 0;
544 }
545
force_reset_timeout(struct intel_engine_cs * engine)546 static void force_reset_timeout(struct intel_engine_cs *engine)
547 {
548 engine->reset_timeout.probability = 999;
549 atomic_set(&engine->reset_timeout.times, -1);
550 }
551
cancel_reset_timeout(struct intel_engine_cs * engine)552 static void cancel_reset_timeout(struct intel_engine_cs *engine)
553 {
554 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
555 }
556
igt_reset_fail_engine(void * arg)557 static int igt_reset_fail_engine(void *arg)
558 {
559 struct intel_gt *gt = arg;
560 struct intel_engine_cs *engine;
561 enum intel_engine_id id;
562
563 /* Check that we can recover from engine-reset failues */
564
565 if (!intel_has_reset_engine(gt))
566 return 0;
567
568 for_each_engine(engine, gt, id) {
569 unsigned int count;
570 struct intel_context *ce;
571 IGT_TIMEOUT(end_time);
572 int err;
573
574 /* Can't manually break the reset if i915 doesn't perform it */
575 if (intel_engine_uses_guc(engine))
576 continue;
577
578 ce = intel_context_create(engine);
579 if (IS_ERR(ce)) {
580 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);
581 return PTR_ERR(ce);
582 }
583
584 st_engine_heartbeat_disable(engine);
585 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
586
587 force_reset_timeout(engine);
588 err = intel_engine_reset(engine, NULL);
589 cancel_reset_timeout(engine);
590 if (err == 0) /* timeouts only generated on gen8+ */
591 goto skip;
592
593 count = 0;
594 do {
595 struct i915_request *last = NULL;
596 int i;
597
598 if (!wait_for_idle(engine)) {
599 pr_err("%s failed to idle before reset\n",
600 engine->name);
601 err = -EIO;
602 break;
603 }
604
605 for (i = 0; i < count % 15; i++) {
606 struct i915_request *rq;
607
608 rq = intel_context_create_request(ce);
609 if (IS_ERR(rq)) {
610 struct drm_printer p =
611 drm_info_printer(gt->i915->drm.dev);
612 intel_engine_dump(engine, &p,
613 "%s(%s): failed to submit request\n",
614 __func__,
615 engine->name);
616
617 GEM_TRACE("%s(%s): failed to submit request\n",
618 __func__,
619 engine->name);
620 GEM_TRACE_DUMP();
621
622 intel_gt_set_wedged(gt);
623 if (last)
624 i915_request_put(last);
625
626 err = PTR_ERR(rq);
627 goto out;
628 }
629
630 if (last)
631 i915_request_put(last);
632 last = i915_request_get(rq);
633 i915_request_add(rq);
634 }
635
636 if (count & 1) {
637 err = intel_engine_reset(engine, NULL);
638 if (err) {
639 GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n",
640 engine->name, err);
641 GEM_TRACE_DUMP();
642 i915_request_put(last);
643 break;
644 }
645 } else {
646 force_reset_timeout(engine);
647 err = intel_engine_reset(engine, NULL);
648 cancel_reset_timeout(engine);
649 if (err != -ETIMEDOUT) {
650 pr_err("intel_engine_reset(%s) did not fail, err:%d\n",
651 engine->name, err);
652 i915_request_put(last);
653 break;
654 }
655 }
656
657 err = 0;
658 if (last) {
659 if (i915_request_wait(last, 0, HZ / 2) < 0) {
660 struct drm_printer p =
661 drm_info_printer(gt->i915->drm.dev);
662
663 intel_engine_dump(engine, &p,
664 "%s(%s): failed to complete request\n",
665 __func__,
666 engine->name);
667
668 GEM_TRACE("%s(%s): failed to complete request\n",
669 __func__,
670 engine->name);
671 GEM_TRACE_DUMP();
672
673 err = -EIO;
674 }
675 i915_request_put(last);
676 }
677 count++;
678 } while (err == 0 && time_before(jiffies, end_time));
679 out:
680 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
681 skip:
682 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
683 st_engine_heartbeat_enable(engine);
684 intel_context_put(ce);
685
686 if (igt_flush_test(gt->i915))
687 err = -EIO;
688 if (err)
689 return err;
690 }
691
692 return 0;
693 }
694
__igt_reset_engine(struct intel_gt * gt,bool active)695 static int __igt_reset_engine(struct intel_gt *gt, bool active)
696 {
697 struct i915_gpu_error *global = >->i915->gpu_error;
698 struct intel_engine_cs *engine;
699 enum intel_engine_id id;
700 struct hang h;
701 int err = 0;
702
703 /* Check that we can issue an engine reset on an idle engine (no-op) */
704
705 if (!intel_has_reset_engine(gt))
706 return 0;
707
708 if (active) {
709 err = hang_init(&h, gt);
710 if (err)
711 return err;
712 }
713
714 for_each_engine(engine, gt, id) {
715 unsigned int reset_count, reset_engine_count;
716 unsigned long count;
717 bool using_guc = intel_engine_uses_guc(engine);
718 IGT_TIMEOUT(end_time);
719
720 if (using_guc && !active)
721 continue;
722
723 if (active && !intel_engine_can_store_dword(engine))
724 continue;
725
726 if (!wait_for_idle(engine)) {
727 pr_err("%s failed to idle before reset\n",
728 engine->name);
729 err = -EIO;
730 break;
731 }
732
733 reset_count = i915_reset_count(global);
734 reset_engine_count = i915_reset_engine_count(global, engine);
735
736 st_engine_heartbeat_disable(engine);
737 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
738 count = 0;
739 do {
740 struct i915_request *rq = NULL;
741 struct intel_selftest_saved_policy saved;
742 int err2;
743
744 err = intel_selftest_modify_policy(engine, &saved,
745 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
746 if (err) {
747 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
748 break;
749 }
750
751 if (active) {
752 rq = hang_create_request(&h, engine);
753 if (IS_ERR(rq)) {
754 err = PTR_ERR(rq);
755 pr_err("[%s] Create hang request failed: %d!\n",
756 engine->name, err);
757 goto restore;
758 }
759
760 i915_request_get(rq);
761 i915_request_add(rq);
762
763 if (!wait_until_running(&h, rq)) {
764 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
765
766 pr_err("%s: Failed to start request %llx, at %x\n",
767 __func__, rq->fence.seqno, hws_seqno(&h, rq));
768 intel_engine_dump(engine, &p,
769 "%s\n", engine->name);
770
771 i915_request_put(rq);
772 err = -EIO;
773 goto restore;
774 }
775 }
776
777 if (!using_guc) {
778 err = intel_engine_reset(engine, NULL);
779 if (err) {
780 pr_err("intel_engine_reset(%s) failed, err:%d\n",
781 engine->name, err);
782 goto skip;
783 }
784 }
785
786 if (rq) {
787 /* Ensure the reset happens and kills the engine */
788 err = intel_selftest_wait_for_rq(rq);
789 if (err)
790 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",
791 engine->name, rq->fence.context,
792 rq->fence.seqno, rq->context->guc_id, err);
793 }
794
795 skip:
796 if (rq)
797 i915_request_put(rq);
798
799 if (i915_reset_count(global) != reset_count) {
800 pr_err("Full GPU reset recorded! (engine reset expected)\n");
801 err = -EINVAL;
802 goto restore;
803 }
804
805 /* GuC based resets are not logged per engine */
806 if (!using_guc) {
807 if (i915_reset_engine_count(global, engine) !=
808 ++reset_engine_count) {
809 pr_err("%s engine reset not recorded!\n",
810 engine->name);
811 err = -EINVAL;
812 goto restore;
813 }
814 }
815
816 count++;
817
818 restore:
819 err2 = intel_selftest_restore_policy(engine, &saved);
820 if (err2)
821 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err);
822 if (err == 0)
823 err = err2;
824 if (err)
825 break;
826 } while (time_before(jiffies, end_time));
827 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
828 st_engine_heartbeat_enable(engine);
829 pr_info("%s: Completed %lu %s resets\n",
830 engine->name, count, active ? "active" : "idle");
831
832 if (err)
833 break;
834
835 err = igt_flush_test(gt->i915);
836 if (err) {
837 pr_err("[%s] Flush failed: %d!\n", engine->name, err);
838 break;
839 }
840 }
841
842 if (intel_gt_is_wedged(gt)) {
843 pr_err("GT is wedged!\n");
844 err = -EIO;
845 }
846
847 if (active)
848 hang_fini(&h);
849
850 return err;
851 }
852
igt_reset_idle_engine(void * arg)853 static int igt_reset_idle_engine(void *arg)
854 {
855 return __igt_reset_engine(arg, false);
856 }
857
igt_reset_active_engine(void * arg)858 static int igt_reset_active_engine(void *arg)
859 {
860 return __igt_reset_engine(arg, true);
861 }
862
863 struct active_engine {
864 struct kthread_worker *worker;
865 struct kthread_work work;
866 struct intel_engine_cs *engine;
867 unsigned long resets;
868 unsigned int flags;
869 bool stop;
870 int result;
871 };
872
873 #define TEST_ACTIVE BIT(0)
874 #define TEST_OTHERS BIT(1)
875 #define TEST_SELF BIT(2)
876 #define TEST_PRIORITY BIT(3)
877
active_request_put(struct i915_request * rq)878 static int active_request_put(struct i915_request *rq)
879 {
880 int err = 0;
881
882 if (!rq)
883 return 0;
884
885 if (i915_request_wait(rq, 0, 10 * HZ) < 0) {
886 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",
887 rq->engine->name,
888 rq->fence.context,
889 rq->fence.seqno);
890 GEM_TRACE_DUMP();
891
892 intel_gt_set_wedged(rq->engine->gt);
893 err = -EIO;
894 }
895
896 i915_request_put(rq);
897
898 return err;
899 }
900
active_engine(struct kthread_work * work)901 static void active_engine(struct kthread_work *work)
902 {
903 I915_RND_STATE(prng);
904 struct active_engine *arg = container_of(work, typeof(*arg), work);
905 struct intel_engine_cs *engine = arg->engine;
906 struct i915_request *rq[8] = {};
907 struct intel_context *ce[ARRAY_SIZE(rq)];
908 unsigned long count;
909 int err = 0;
910
911 for (count = 0; count < ARRAY_SIZE(ce); count++) {
912 ce[count] = intel_context_create(engine);
913 if (IS_ERR(ce[count])) {
914 arg->result = PTR_ERR(ce[count]);
915 pr_err("[%s] Create context #%ld failed: %d!\n",
916 engine->name, count, arg->result);
917 while (--count)
918 intel_context_put(ce[count]);
919 return;
920 }
921 }
922
923 count = 0;
924 while (!READ_ONCE(arg->stop)) {
925 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
926 struct i915_request *old = rq[idx];
927 struct i915_request *new;
928
929 new = intel_context_create_request(ce[idx]);
930 if (IS_ERR(new)) {
931 err = PTR_ERR(new);
932 pr_err("[%s] Create request #%d failed: %d!\n", engine->name, idx, err);
933 break;
934 }
935
936 rq[idx] = i915_request_get(new);
937 i915_request_add(new);
938
939 if (engine->sched_engine->schedule && arg->flags & TEST_PRIORITY) {
940 struct i915_sched_attr attr = {
941 .priority =
942 i915_prandom_u32_max_state(512, &prng),
943 };
944 engine->sched_engine->schedule(rq[idx], &attr);
945 }
946
947 err = active_request_put(old);
948 if (err) {
949 pr_err("[%s] Request put failed: %d!\n", engine->name, err);
950 break;
951 }
952
953 cond_resched();
954 }
955
956 for (count = 0; count < ARRAY_SIZE(rq); count++) {
957 int err__ = active_request_put(rq[count]);
958
959 if (err)
960 pr_err("[%s] Request put #%ld failed: %d!\n", engine->name, count, err);
961
962 /* Keep the first error */
963 if (!err)
964 err = err__;
965
966 intel_context_put(ce[count]);
967 }
968
969 arg->result = err;
970 }
971
__igt_reset_engines(struct intel_gt * gt,const char * test_name,unsigned int flags)972 static int __igt_reset_engines(struct intel_gt *gt,
973 const char *test_name,
974 unsigned int flags)
975 {
976 struct i915_gpu_error *global = >->i915->gpu_error;
977 struct intel_engine_cs *engine, *other;
978 enum intel_engine_id id, tmp;
979 struct hang h;
980 int err = 0;
981
982 /* Check that issuing a reset on one engine does not interfere
983 * with any other engine.
984 */
985
986 if (!intel_has_reset_engine(gt))
987 return 0;
988
989 if (flags & TEST_ACTIVE) {
990 err = hang_init(&h, gt);
991 if (err)
992 return err;
993
994 if (flags & TEST_PRIORITY)
995 h.ctx->sched.priority = 1024;
996 }
997
998 for_each_engine(engine, gt, id) {
999 struct active_engine threads[I915_NUM_ENGINES] = {};
1000 unsigned long device = i915_reset_count(global);
1001 unsigned long count = 0, reported;
1002 bool using_guc = intel_engine_uses_guc(engine);
1003 IGT_TIMEOUT(end_time);
1004
1005 if (flags & TEST_ACTIVE) {
1006 if (!intel_engine_can_store_dword(engine))
1007 continue;
1008 } else if (using_guc)
1009 continue;
1010
1011 if (!wait_for_idle(engine)) {
1012 pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
1013 engine->name, test_name);
1014 err = -EIO;
1015 break;
1016 }
1017
1018 memset(threads, 0, sizeof(threads));
1019 for_each_engine(other, gt, tmp) {
1020 struct kthread_worker *worker;
1021
1022 threads[tmp].resets =
1023 i915_reset_engine_count(global, other);
1024
1025 if (other == engine && !(flags & TEST_SELF))
1026 continue;
1027
1028 if (other != engine && !(flags & TEST_OTHERS))
1029 continue;
1030
1031 threads[tmp].engine = other;
1032 threads[tmp].flags = flags;
1033
1034 worker = kthread_create_worker(0, "igt/%s",
1035 other->name);
1036 if (IS_ERR(worker)) {
1037 err = PTR_ERR(worker);
1038 pr_err("[%s] Worker create failed: %d!\n",
1039 engine->name, err);
1040 goto unwind;
1041 }
1042
1043 threads[tmp].worker = worker;
1044
1045 kthread_init_work(&threads[tmp].work, active_engine);
1046 kthread_queue_work(threads[tmp].worker,
1047 &threads[tmp].work);
1048 }
1049
1050 st_engine_heartbeat_disable_no_pm(engine);
1051 set_bit(I915_RESET_ENGINE + id, >->reset.flags);
1052 do {
1053 struct i915_request *rq = NULL;
1054 struct intel_selftest_saved_policy saved;
1055 int err2;
1056
1057 err = intel_selftest_modify_policy(engine, &saved,
1058 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1059 if (err) {
1060 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
1061 break;
1062 }
1063
1064 if (flags & TEST_ACTIVE) {
1065 rq = hang_create_request(&h, engine);
1066 if (IS_ERR(rq)) {
1067 err = PTR_ERR(rq);
1068 pr_err("[%s] Create hang request failed: %d!\n",
1069 engine->name, err);
1070 goto restore;
1071 }
1072
1073 i915_request_get(rq);
1074 i915_request_add(rq);
1075
1076 if (!wait_until_running(&h, rq)) {
1077 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1078
1079 pr_err("%s: Failed to start request %llx, at %x\n",
1080 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1081 intel_engine_dump(engine, &p,
1082 "%s\n", engine->name);
1083
1084 i915_request_put(rq);
1085 err = -EIO;
1086 goto restore;
1087 }
1088 } else {
1089 intel_engine_pm_get(engine);
1090 }
1091
1092 if (!using_guc) {
1093 err = intel_engine_reset(engine, NULL);
1094 if (err) {
1095 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
1096 engine->name, test_name, err);
1097 goto restore;
1098 }
1099 }
1100
1101 if (rq) {
1102 /* Ensure the reset happens and kills the engine */
1103 err = intel_selftest_wait_for_rq(rq);
1104 if (err)
1105 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",
1106 engine->name, rq->fence.context,
1107 rq->fence.seqno, rq->context->guc_id, err);
1108 }
1109
1110 count++;
1111
1112 if (rq) {
1113 if (rq->fence.error != -EIO) {
1114 pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n",
1115 engine->name, test_name,
1116 rq->fence.context,
1117 rq->fence.seqno, rq->context->guc_id);
1118 i915_request_put(rq);
1119
1120 GEM_TRACE_DUMP();
1121 intel_gt_set_wedged(gt);
1122 err = -EIO;
1123 goto restore;
1124 }
1125
1126 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1127 struct drm_printer p =
1128 drm_info_printer(gt->i915->drm.dev);
1129
1130 pr_err("i915_reset_engine(%s:%s):"
1131 " failed to complete request %llx:%lld after reset\n",
1132 engine->name, test_name,
1133 rq->fence.context,
1134 rq->fence.seqno);
1135 intel_engine_dump(engine, &p,
1136 "%s\n", engine->name);
1137 i915_request_put(rq);
1138
1139 GEM_TRACE_DUMP();
1140 intel_gt_set_wedged(gt);
1141 err = -EIO;
1142 goto restore;
1143 }
1144
1145 i915_request_put(rq);
1146 }
1147
1148 if (!(flags & TEST_ACTIVE))
1149 intel_engine_pm_put(engine);
1150
1151 if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
1152 struct drm_printer p =
1153 drm_info_printer(gt->i915->drm.dev);
1154
1155 pr_err("i915_reset_engine(%s:%s):"
1156 " failed to idle after reset\n",
1157 engine->name, test_name);
1158 intel_engine_dump(engine, &p,
1159 "%s\n", engine->name);
1160
1161 err = -EIO;
1162 goto restore;
1163 }
1164
1165 restore:
1166 err2 = intel_selftest_restore_policy(engine, &saved);
1167 if (err2)
1168 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err2);
1169 if (err == 0)
1170 err = err2;
1171 if (err)
1172 break;
1173 } while (time_before(jiffies, end_time));
1174 clear_bit(I915_RESET_ENGINE + id, >->reset.flags);
1175 st_engine_heartbeat_enable_no_pm(engine);
1176
1177 pr_info("i915_reset_engine(%s:%s): %lu resets\n",
1178 engine->name, test_name, count);
1179
1180 /* GuC based resets are not logged per engine */
1181 if (!using_guc) {
1182 reported = i915_reset_engine_count(global, engine);
1183 reported -= threads[engine->id].resets;
1184 if (reported != count) {
1185 pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
1186 engine->name, test_name, count, reported);
1187 if (!err)
1188 err = -EINVAL;
1189 }
1190 }
1191
1192 unwind:
1193 for_each_engine(other, gt, tmp) {
1194 int ret;
1195
1196 if (!threads[tmp].worker)
1197 continue;
1198
1199 WRITE_ONCE(threads[tmp].stop, true);
1200 kthread_flush_work(&threads[tmp].work);
1201 ret = READ_ONCE(threads[tmp].result);
1202 if (ret) {
1203 pr_err("kthread for other engine %s failed, err=%d\n",
1204 other->name, ret);
1205 if (!err)
1206 err = ret;
1207 }
1208
1209 kthread_destroy_worker(threads[tmp].worker);
1210
1211 /* GuC based resets are not logged per engine */
1212 if (!using_guc) {
1213 if (other->uabi_class != engine->uabi_class &&
1214 threads[tmp].resets !=
1215 i915_reset_engine_count(global, other)) {
1216 pr_err("Innocent engine %s was reset (count=%ld)\n",
1217 other->name,
1218 i915_reset_engine_count(global, other) -
1219 threads[tmp].resets);
1220 if (!err)
1221 err = -EINVAL;
1222 }
1223 }
1224 }
1225
1226 if (device != i915_reset_count(global)) {
1227 pr_err("Global reset (count=%ld)!\n",
1228 i915_reset_count(global) - device);
1229 if (!err)
1230 err = -EINVAL;
1231 }
1232
1233 if (err)
1234 break;
1235
1236 err = igt_flush_test(gt->i915);
1237 if (err) {
1238 pr_err("[%s] Flush failed: %d!\n", engine->name, err);
1239 break;
1240 }
1241 }
1242
1243 if (intel_gt_is_wedged(gt))
1244 err = -EIO;
1245
1246 if (flags & TEST_ACTIVE)
1247 hang_fini(&h);
1248
1249 return err;
1250 }
1251
igt_reset_engines(void * arg)1252 static int igt_reset_engines(void *arg)
1253 {
1254 static const struct {
1255 const char *name;
1256 unsigned int flags;
1257 } phases[] = {
1258 { "idle", 0 },
1259 { "active", TEST_ACTIVE },
1260 { "others-idle", TEST_OTHERS },
1261 { "others-active", TEST_OTHERS | TEST_ACTIVE },
1262 {
1263 "others-priority",
1264 TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
1265 },
1266 {
1267 "self-priority",
1268 TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
1269 },
1270 { }
1271 };
1272 struct intel_gt *gt = arg;
1273 typeof(*phases) *p;
1274 int err;
1275
1276 for (p = phases; p->name; p++) {
1277 if (p->flags & TEST_PRIORITY) {
1278 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
1279 continue;
1280 }
1281
1282 err = __igt_reset_engines(arg, p->name, p->flags);
1283 if (err)
1284 return err;
1285 }
1286
1287 return 0;
1288 }
1289
fake_hangcheck(struct intel_gt * gt,intel_engine_mask_t mask)1290 static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask)
1291 {
1292 u32 count = i915_reset_count(>->i915->gpu_error);
1293
1294 intel_gt_reset(gt, mask, NULL);
1295
1296 return count;
1297 }
1298
igt_reset_wait(void * arg)1299 static int igt_reset_wait(void *arg)
1300 {
1301 struct intel_gt *gt = arg;
1302 struct i915_gpu_error *global = >->i915->gpu_error;
1303 struct intel_engine_cs *engine = gt->engine[RCS0];
1304 struct i915_request *rq;
1305 unsigned int reset_count;
1306 struct hang h;
1307 long timeout;
1308 int err;
1309
1310 if (!engine || !intel_engine_can_store_dword(engine))
1311 return 0;
1312
1313 /* Check that we detect a stuck waiter and issue a reset */
1314
1315 igt_global_reset_lock(gt);
1316
1317 err = hang_init(&h, gt);
1318 if (err) {
1319 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1320 goto unlock;
1321 }
1322
1323 rq = hang_create_request(&h, engine);
1324 if (IS_ERR(rq)) {
1325 err = PTR_ERR(rq);
1326 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1327 goto fini;
1328 }
1329
1330 i915_request_get(rq);
1331 i915_request_add(rq);
1332
1333 if (!wait_until_running(&h, rq)) {
1334 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1335
1336 pr_err("%s: Failed to start request %llx, at %x\n",
1337 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1338 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1339
1340 intel_gt_set_wedged(gt);
1341
1342 err = -EIO;
1343 goto out_rq;
1344 }
1345
1346 reset_count = fake_hangcheck(gt, ALL_ENGINES);
1347
1348 timeout = i915_request_wait(rq, 0, 10);
1349 if (timeout < 0) {
1350 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
1351 timeout);
1352 err = timeout;
1353 goto out_rq;
1354 }
1355
1356 if (i915_reset_count(global) == reset_count) {
1357 pr_err("No GPU reset recorded!\n");
1358 err = -EINVAL;
1359 goto out_rq;
1360 }
1361
1362 out_rq:
1363 i915_request_put(rq);
1364 fini:
1365 hang_fini(&h);
1366 unlock:
1367 igt_global_reset_unlock(gt);
1368
1369 if (intel_gt_is_wedged(gt))
1370 return -EIO;
1371
1372 return err;
1373 }
1374
1375 struct evict_vma {
1376 struct completion completion;
1377 struct i915_vma *vma;
1378 };
1379
evict_vma(void * data)1380 static int evict_vma(void *data)
1381 {
1382 struct evict_vma *arg = data;
1383 struct i915_address_space *vm = arg->vma->vm;
1384 struct drm_mm_node evict = arg->vma->node;
1385 int err;
1386
1387 complete(&arg->completion);
1388
1389 mutex_lock(&vm->mutex);
1390 err = i915_gem_evict_for_node(vm, &evict, 0);
1391 mutex_unlock(&vm->mutex);
1392
1393 return err;
1394 }
1395
evict_fence(void * data)1396 static int evict_fence(void *data)
1397 {
1398 struct evict_vma *arg = data;
1399 int err;
1400
1401 complete(&arg->completion);
1402
1403 /* Mark the fence register as dirty to force the mmio update. */
1404 err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
1405 if (err) {
1406 pr_err("Invalid Y-tiling settings; err:%d\n", err);
1407 return err;
1408 }
1409
1410 err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);
1411 if (err) {
1412 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);
1413 return err;
1414 }
1415
1416 err = i915_vma_pin_fence(arg->vma);
1417 i915_vma_unpin(arg->vma);
1418 if (err) {
1419 pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
1420 return err;
1421 }
1422
1423 i915_vma_unpin_fence(arg->vma);
1424
1425 return 0;
1426 }
1427
__igt_reset_evict_vma(struct intel_gt * gt,struct i915_address_space * vm,int (* fn)(void *),unsigned int flags)1428 static int __igt_reset_evict_vma(struct intel_gt *gt,
1429 struct i915_address_space *vm,
1430 int (*fn)(void *),
1431 unsigned int flags)
1432 {
1433 struct intel_engine_cs *engine = gt->engine[RCS0];
1434 struct drm_i915_gem_object *obj;
1435 struct task_struct *tsk = NULL;
1436 struct i915_request *rq;
1437 struct evict_vma arg;
1438 struct hang h;
1439 unsigned int pin_flags;
1440 int err;
1441
1442 if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)
1443 return 0;
1444
1445 if (!engine || !intel_engine_can_store_dword(engine))
1446 return 0;
1447
1448 /* Check that we can recover an unbind stuck on a hanging request */
1449
1450 err = hang_init(&h, gt);
1451 if (err) {
1452 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1453 return err;
1454 }
1455
1456 obj = i915_gem_object_create_internal(gt->i915, SZ_1M);
1457 if (IS_ERR(obj)) {
1458 err = PTR_ERR(obj);
1459 pr_err("[%s] Create object failed: %d!\n", engine->name, err);
1460 goto fini;
1461 }
1462
1463 if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1464 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
1465 if (err) {
1466 pr_err("Invalid X-tiling settings; err:%d\n", err);
1467 goto out_obj;
1468 }
1469 }
1470
1471 arg.vma = i915_vma_instance(obj, vm, NULL);
1472 if (IS_ERR(arg.vma)) {
1473 err = PTR_ERR(arg.vma);
1474 pr_err("[%s] VMA instance failed: %d!\n", engine->name, err);
1475 goto out_obj;
1476 }
1477
1478 rq = hang_create_request(&h, engine);
1479 if (IS_ERR(rq)) {
1480 err = PTR_ERR(rq);
1481 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1482 goto out_obj;
1483 }
1484
1485 pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;
1486
1487 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1488 pin_flags |= PIN_MAPPABLE;
1489
1490 err = i915_vma_pin(arg.vma, 0, 0, pin_flags);
1491 if (err) {
1492 i915_request_add(rq);
1493 pr_err("[%s] VMA pin failed: %d!\n", engine->name, err);
1494 goto out_obj;
1495 }
1496
1497 if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1498 err = i915_vma_pin_fence(arg.vma);
1499 if (err) {
1500 pr_err("Unable to pin X-tiled fence; err:%d\n", err);
1501 i915_vma_unpin(arg.vma);
1502 i915_request_add(rq);
1503 goto out_obj;
1504 }
1505 }
1506
1507 i915_vma_lock(arg.vma);
1508 err = i915_request_await_object(rq, arg.vma->obj,
1509 flags & EXEC_OBJECT_WRITE);
1510 if (err == 0) {
1511 err = i915_vma_move_to_active(arg.vma, rq, flags);
1512 if (err)
1513 pr_err("[%s] Move to active failed: %d!\n", engine->name, err);
1514 } else {
1515 pr_err("[%s] Request await failed: %d!\n", engine->name, err);
1516 }
1517
1518 i915_vma_unlock(arg.vma);
1519
1520 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1521 i915_vma_unpin_fence(arg.vma);
1522 i915_vma_unpin(arg.vma);
1523
1524 i915_request_get(rq);
1525 i915_request_add(rq);
1526 if (err)
1527 goto out_rq;
1528
1529 if (!wait_until_running(&h, rq)) {
1530 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1531
1532 pr_err("%s: Failed to start request %llx, at %x\n",
1533 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1534 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1535
1536 intel_gt_set_wedged(gt);
1537 goto out_reset;
1538 }
1539
1540 init_completion(&arg.completion);
1541
1542 tsk = kthread_run(fn, &arg, "igt/evict_vma");
1543 if (IS_ERR(tsk)) {
1544 err = PTR_ERR(tsk);
1545 pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err);
1546 tsk = NULL;
1547 goto out_reset;
1548 }
1549 get_task_struct(tsk);
1550
1551 wait_for_completion(&arg.completion);
1552
1553 if (wait_for(!list_empty(&rq->fence.cb_list), 10)) {
1554 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1555
1556 pr_err("igt/evict_vma kthread did not wait\n");
1557 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1558
1559 intel_gt_set_wedged(gt);
1560 goto out_reset;
1561 }
1562
1563 out_reset:
1564 igt_global_reset_lock(gt);
1565 fake_hangcheck(gt, rq->engine->mask);
1566 igt_global_reset_unlock(gt);
1567
1568 if (tsk) {
1569 struct intel_wedge_me w;
1570
1571 /* The reset, even indirectly, should take less than 10ms. */
1572 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
1573 err = kthread_stop(tsk);
1574
1575 put_task_struct(tsk);
1576 }
1577
1578 out_rq:
1579 i915_request_put(rq);
1580 out_obj:
1581 i915_gem_object_put(obj);
1582 fini:
1583 hang_fini(&h);
1584 if (intel_gt_is_wedged(gt))
1585 return -EIO;
1586
1587 return err;
1588 }
1589
igt_reset_evict_ggtt(void * arg)1590 static int igt_reset_evict_ggtt(void *arg)
1591 {
1592 struct intel_gt *gt = arg;
1593
1594 return __igt_reset_evict_vma(gt, >->ggtt->vm,
1595 evict_vma, EXEC_OBJECT_WRITE);
1596 }
1597
igt_reset_evict_ppgtt(void * arg)1598 static int igt_reset_evict_ppgtt(void *arg)
1599 {
1600 struct intel_gt *gt = arg;
1601 struct i915_ppgtt *ppgtt;
1602 int err;
1603
1604 /* aliasing == global gtt locking, covered above */
1605 if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL)
1606 return 0;
1607
1608 ppgtt = i915_ppgtt_create(gt);
1609 if (IS_ERR(ppgtt))
1610 return PTR_ERR(ppgtt);
1611
1612 err = __igt_reset_evict_vma(gt, &ppgtt->vm,
1613 evict_vma, EXEC_OBJECT_WRITE);
1614 i915_vm_put(&ppgtt->vm);
1615
1616 return err;
1617 }
1618
igt_reset_evict_fence(void * arg)1619 static int igt_reset_evict_fence(void *arg)
1620 {
1621 struct intel_gt *gt = arg;
1622
1623 return __igt_reset_evict_vma(gt, >->ggtt->vm,
1624 evict_fence, EXEC_OBJECT_NEEDS_FENCE);
1625 }
1626
wait_for_others(struct intel_gt * gt,struct intel_engine_cs * exclude)1627 static int wait_for_others(struct intel_gt *gt,
1628 struct intel_engine_cs *exclude)
1629 {
1630 struct intel_engine_cs *engine;
1631 enum intel_engine_id id;
1632
1633 for_each_engine(engine, gt, id) {
1634 if (engine == exclude)
1635 continue;
1636
1637 if (!wait_for_idle(engine))
1638 return -EIO;
1639 }
1640
1641 return 0;
1642 }
1643
igt_reset_queue(void * arg)1644 static int igt_reset_queue(void *arg)
1645 {
1646 struct intel_gt *gt = arg;
1647 struct i915_gpu_error *global = >->i915->gpu_error;
1648 struct intel_engine_cs *engine;
1649 enum intel_engine_id id;
1650 struct hang h;
1651 int err;
1652
1653 /* Check that we replay pending requests following a hang */
1654
1655 igt_global_reset_lock(gt);
1656
1657 err = hang_init(&h, gt);
1658 if (err)
1659 goto unlock;
1660
1661 for_each_engine(engine, gt, id) {
1662 struct intel_selftest_saved_policy saved;
1663 struct i915_request *prev;
1664 IGT_TIMEOUT(end_time);
1665 unsigned int count;
1666 bool using_guc = intel_engine_uses_guc(engine);
1667
1668 if (!intel_engine_can_store_dword(engine))
1669 continue;
1670
1671 if (using_guc) {
1672 err = intel_selftest_modify_policy(engine, &saved,
1673 SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK);
1674 if (err) {
1675 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
1676 goto fini;
1677 }
1678 }
1679
1680 prev = hang_create_request(&h, engine);
1681 if (IS_ERR(prev)) {
1682 err = PTR_ERR(prev);
1683 pr_err("[%s] Create 'prev' hang request failed: %d!\n", engine->name, err);
1684 goto restore;
1685 }
1686
1687 i915_request_get(prev);
1688 i915_request_add(prev);
1689
1690 count = 0;
1691 do {
1692 struct i915_request *rq;
1693 unsigned int reset_count;
1694
1695 rq = hang_create_request(&h, engine);
1696 if (IS_ERR(rq)) {
1697 err = PTR_ERR(rq);
1698 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1699 goto restore;
1700 }
1701
1702 i915_request_get(rq);
1703 i915_request_add(rq);
1704
1705 /*
1706 * XXX We don't handle resetting the kernel context
1707 * very well. If we trigger a device reset twice in
1708 * quick succession while the kernel context is
1709 * executing, we may end up skipping the breadcrumb.
1710 * This is really only a problem for the selftest as
1711 * normally there is a large interlude between resets
1712 * (hangcheck), or we focus on resetting just one
1713 * engine and so avoid repeatedly resetting innocents.
1714 */
1715 err = wait_for_others(gt, engine);
1716 if (err) {
1717 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
1718 __func__, engine->name);
1719 i915_request_put(rq);
1720 i915_request_put(prev);
1721
1722 GEM_TRACE_DUMP();
1723 intel_gt_set_wedged(gt);
1724 goto restore;
1725 }
1726
1727 if (!wait_until_running(&h, prev)) {
1728 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1729
1730 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1731 __func__, engine->name,
1732 prev->fence.seqno, hws_seqno(&h, prev));
1733 intel_engine_dump(engine, &p,
1734 "%s\n", engine->name);
1735
1736 i915_request_put(rq);
1737 i915_request_put(prev);
1738
1739 intel_gt_set_wedged(gt);
1740
1741 err = -EIO;
1742 goto restore;
1743 }
1744
1745 reset_count = fake_hangcheck(gt, BIT(id));
1746
1747 if (prev->fence.error != -EIO) {
1748 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
1749 prev->fence.error);
1750 i915_request_put(rq);
1751 i915_request_put(prev);
1752 err = -EINVAL;
1753 goto restore;
1754 }
1755
1756 if (rq->fence.error) {
1757 pr_err("Fence error status not zero [%d] after unrelated reset\n",
1758 rq->fence.error);
1759 i915_request_put(rq);
1760 i915_request_put(prev);
1761 err = -EINVAL;
1762 goto restore;
1763 }
1764
1765 if (i915_reset_count(global) == reset_count) {
1766 pr_err("No GPU reset recorded!\n");
1767 i915_request_put(rq);
1768 i915_request_put(prev);
1769 err = -EINVAL;
1770 goto restore;
1771 }
1772
1773 i915_request_put(prev);
1774 prev = rq;
1775 count++;
1776 } while (time_before(jiffies, end_time));
1777 pr_info("%s: Completed %d queued resets\n",
1778 engine->name, count);
1779
1780 *h.batch = MI_BATCH_BUFFER_END;
1781 intel_gt_chipset_flush(engine->gt);
1782
1783 i915_request_put(prev);
1784
1785 restore:
1786 if (using_guc) {
1787 int err2 = intel_selftest_restore_policy(engine, &saved);
1788
1789 if (err2)
1790 pr_err("%s:%d> [%s] Restore policy failed: %d!\n",
1791 __func__, __LINE__, engine->name, err2);
1792 if (err == 0)
1793 err = err2;
1794 }
1795 if (err)
1796 goto fini;
1797
1798 err = igt_flush_test(gt->i915);
1799 if (err) {
1800 pr_err("[%s] Flush failed: %d!\n", engine->name, err);
1801 break;
1802 }
1803 }
1804
1805 fini:
1806 hang_fini(&h);
1807 unlock:
1808 igt_global_reset_unlock(gt);
1809
1810 if (intel_gt_is_wedged(gt))
1811 return -EIO;
1812
1813 return err;
1814 }
1815
igt_handle_error(void * arg)1816 static int igt_handle_error(void *arg)
1817 {
1818 struct intel_gt *gt = arg;
1819 struct i915_gpu_error *global = >->i915->gpu_error;
1820 struct intel_engine_cs *engine = gt->engine[RCS0];
1821 struct hang h;
1822 struct i915_request *rq;
1823 struct i915_gpu_coredump *error;
1824 int err;
1825
1826 /* Check that we can issue a global GPU and engine reset */
1827
1828 if (!intel_has_reset_engine(gt))
1829 return 0;
1830
1831 if (!engine || !intel_engine_can_store_dword(engine))
1832 return 0;
1833
1834 err = hang_init(&h, gt);
1835 if (err) {
1836 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1837 return err;
1838 }
1839
1840 rq = hang_create_request(&h, engine);
1841 if (IS_ERR(rq)) {
1842 err = PTR_ERR(rq);
1843 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1844 goto err_fini;
1845 }
1846
1847 i915_request_get(rq);
1848 i915_request_add(rq);
1849
1850 if (!wait_until_running(&h, rq)) {
1851 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1852
1853 pr_err("%s: Failed to start request %llx, at %x\n",
1854 __func__, rq->fence.seqno, hws_seqno(&h, rq));
1855 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1856
1857 intel_gt_set_wedged(gt);
1858
1859 err = -EIO;
1860 goto err_request;
1861 }
1862
1863 /* Temporarily disable error capture */
1864 error = xchg(&global->first_error, (void *)-1);
1865
1866 intel_gt_handle_error(gt, engine->mask, 0, NULL);
1867
1868 xchg(&global->first_error, error);
1869
1870 if (rq->fence.error != -EIO) {
1871 pr_err("Guilty request not identified!\n");
1872 err = -EINVAL;
1873 goto err_request;
1874 }
1875
1876 err_request:
1877 i915_request_put(rq);
1878 err_fini:
1879 hang_fini(&h);
1880 return err;
1881 }
1882
__igt_atomic_reset_engine(struct intel_engine_cs * engine,const struct igt_atomic_section * p,const char * mode)1883 static int __igt_atomic_reset_engine(struct intel_engine_cs *engine,
1884 const struct igt_atomic_section *p,
1885 const char *mode)
1886 {
1887 struct tasklet_struct * const t = &engine->sched_engine->tasklet;
1888 int err;
1889
1890 GEM_TRACE("i915_reset_engine(%s:%s) under %s\n",
1891 engine->name, mode, p->name);
1892
1893 if (t->func)
1894 tasklet_disable(t);
1895 if (strcmp(p->name, "softirq"))
1896 local_bh_disable();
1897 p->critical_section_begin();
1898
1899 err = __intel_engine_reset_bh(engine, NULL);
1900
1901 p->critical_section_end();
1902 if (strcmp(p->name, "softirq"))
1903 local_bh_enable();
1904 if (t->func) {
1905 tasklet_enable(t);
1906 tasklet_hi_schedule(t);
1907 }
1908
1909 if (err)
1910 pr_err("i915_reset_engine(%s:%s) failed under %s\n",
1911 engine->name, mode, p->name);
1912
1913 return err;
1914 }
1915
igt_atomic_reset_engine(struct intel_engine_cs * engine,const struct igt_atomic_section * p)1916 static int igt_atomic_reset_engine(struct intel_engine_cs *engine,
1917 const struct igt_atomic_section *p)
1918 {
1919 struct i915_request *rq;
1920 struct hang h;
1921 int err;
1922
1923 err = __igt_atomic_reset_engine(engine, p, "idle");
1924 if (err)
1925 return err;
1926
1927 err = hang_init(&h, engine->gt);
1928 if (err) {
1929 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1930 return err;
1931 }
1932
1933 rq = hang_create_request(&h, engine);
1934 if (IS_ERR(rq)) {
1935 err = PTR_ERR(rq);
1936 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1937 goto out;
1938 }
1939
1940 i915_request_get(rq);
1941 i915_request_add(rq);
1942
1943 if (wait_until_running(&h, rq)) {
1944 err = __igt_atomic_reset_engine(engine, p, "active");
1945 } else {
1946 pr_err("%s(%s): Failed to start request %llx, at %x\n",
1947 __func__, engine->name,
1948 rq->fence.seqno, hws_seqno(&h, rq));
1949 intel_gt_set_wedged(engine->gt);
1950 err = -EIO;
1951 }
1952
1953 if (err == 0) {
1954 struct intel_wedge_me w;
1955
1956 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */)
1957 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1958 if (intel_gt_is_wedged(engine->gt))
1959 err = -EIO;
1960 }
1961
1962 i915_request_put(rq);
1963 out:
1964 hang_fini(&h);
1965 return err;
1966 }
1967
igt_reset_engines_atomic(void * arg)1968 static int igt_reset_engines_atomic(void *arg)
1969 {
1970 struct intel_gt *gt = arg;
1971 const typeof(*igt_atomic_phases) *p;
1972 int err = 0;
1973
1974 /* Check that the engines resets are usable from atomic context */
1975
1976 if (!intel_has_reset_engine(gt))
1977 return 0;
1978
1979 if (intel_uc_uses_guc_submission(>->uc))
1980 return 0;
1981
1982 igt_global_reset_lock(gt);
1983
1984 /* Flush any requests before we get started and check basics */
1985 if (!igt_force_reset(gt))
1986 goto unlock;
1987
1988 for (p = igt_atomic_phases; p->name; p++) {
1989 struct intel_engine_cs *engine;
1990 enum intel_engine_id id;
1991
1992 for_each_engine(engine, gt, id) {
1993 err = igt_atomic_reset_engine(engine, p);
1994 if (err)
1995 goto out;
1996 }
1997 }
1998
1999 out:
2000 /* As we poke around the guts, do a full reset before continuing. */
2001 igt_force_reset(gt);
2002 unlock:
2003 igt_global_reset_unlock(gt);
2004
2005 return err;
2006 }
2007
intel_hangcheck_live_selftests(struct drm_i915_private * i915)2008 int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
2009 {
2010 static const struct i915_subtest tests[] = {
2011 SUBTEST(igt_hang_sanitycheck),
2012 SUBTEST(igt_reset_nop),
2013 SUBTEST(igt_reset_nop_engine),
2014 SUBTEST(igt_reset_idle_engine),
2015 SUBTEST(igt_reset_active_engine),
2016 SUBTEST(igt_reset_fail_engine),
2017 SUBTEST(igt_reset_engines),
2018 SUBTEST(igt_reset_engines_atomic),
2019 SUBTEST(igt_reset_queue),
2020 SUBTEST(igt_reset_wait),
2021 SUBTEST(igt_reset_evict_ggtt),
2022 SUBTEST(igt_reset_evict_ppgtt),
2023 SUBTEST(igt_reset_evict_fence),
2024 SUBTEST(igt_handle_error),
2025 };
2026 struct intel_gt *gt = &i915->gt;
2027 intel_wakeref_t wakeref;
2028 int err;
2029
2030 if (!intel_has_gpu_reset(gt))
2031 return 0;
2032
2033 if (intel_gt_is_wedged(gt))
2034 return -EIO; /* we're long past hope of a successful reset */
2035
2036 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
2037
2038 err = intel_gt_live_subtests(tests, gt);
2039
2040 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
2041
2042 return err;
2043 }
2044