1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_gt.h"
7
8 #include <linux/minmax.h>
9
10 #include <drm/drm_managed.h>
11 #include <uapi/drm/xe_drm.h>
12
13 #include <generated/xe_wa_oob.h>
14
15 #include "instructions/xe_gfxpipe_commands.h"
16 #include "instructions/xe_mi_commands.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_assert.h"
19 #include "xe_bb.h"
20 #include "xe_bo.h"
21 #include "xe_device.h"
22 #include "xe_exec_queue.h"
23 #include "xe_execlist.h"
24 #include "xe_force_wake.h"
25 #include "xe_ggtt.h"
26 #include "xe_gsc.h"
27 #include "xe_gt_ccs_mode.h"
28 #include "xe_gt_clock.h"
29 #include "xe_gt_freq.h"
30 #include "xe_gt_idle.h"
31 #include "xe_gt_mcr.h"
32 #include "xe_gt_pagefault.h"
33 #include "xe_gt_printk.h"
34 #include "xe_gt_sriov_pf.h"
35 #include "xe_gt_sriov_vf.h"
36 #include "xe_gt_sysfs.h"
37 #include "xe_gt_tlb_invalidation.h"
38 #include "xe_gt_topology.h"
39 #include "xe_guc_exec_queue_types.h"
40 #include "xe_guc_pc.h"
41 #include "xe_hw_fence.h"
42 #include "xe_hw_engine_class_sysfs.h"
43 #include "xe_irq.h"
44 #include "xe_lmtt.h"
45 #include "xe_lrc.h"
46 #include "xe_map.h"
47 #include "xe_migrate.h"
48 #include "xe_mmio.h"
49 #include "xe_pat.h"
50 #include "xe_pm.h"
51 #include "xe_mocs.h"
52 #include "xe_reg_sr.h"
53 #include "xe_ring_ops.h"
54 #include "xe_sa.h"
55 #include "xe_sched_job.h"
56 #include "xe_sriov.h"
57 #include "xe_tuning.h"
58 #include "xe_uc.h"
59 #include "xe_uc_fw.h"
60 #include "xe_vm.h"
61 #include "xe_wa.h"
62 #include "xe_wopcm.h"
63
gt_fini(struct drm_device * drm,void * arg)64 static void gt_fini(struct drm_device *drm, void *arg)
65 {
66 struct xe_gt *gt = arg;
67
68 destroy_workqueue(gt->ordered_wq);
69 }
70
xe_gt_alloc(struct xe_tile * tile)71 struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
72 {
73 struct xe_gt *gt;
74 int err;
75
76 gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL);
77 if (!gt)
78 return ERR_PTR(-ENOMEM);
79
80 gt->tile = tile;
81 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0);
82
83 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt);
84 if (err)
85 return ERR_PTR(err);
86
87 return gt;
88 }
89
xe_gt_sanitize(struct xe_gt * gt)90 void xe_gt_sanitize(struct xe_gt *gt)
91 {
92 /*
93 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
94 * reload
95 */
96 gt->uc.guc.submission_state.enabled = false;
97 }
98
xe_gt_enable_host_l2_vram(struct xe_gt * gt)99 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
100 {
101 u32 reg;
102 int err;
103
104 if (!XE_WA(gt, 16023588340))
105 return;
106
107 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
108 if (WARN_ON(err))
109 return;
110
111 if (!xe_gt_is_media_type(gt)) {
112 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
113 reg |= CG_DIS_CNTLBUS;
114 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
115 }
116
117 xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
118 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
119 }
120
xe_gt_disable_host_l2_vram(struct xe_gt * gt)121 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
122 {
123 u32 reg;
124 int err;
125
126 if (!XE_WA(gt, 16023588340))
127 return;
128
129 if (xe_gt_is_media_type(gt))
130 return;
131
132 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
133 if (WARN_ON(err))
134 return;
135
136 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
137 reg &= ~CG_DIS_CNTLBUS;
138 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
139
140 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
141 }
142
143 /**
144 * xe_gt_remove() - Clean up the GT structures before driver removal
145 * @gt: the GT object
146 *
147 * This function should only act on objects/structures that must be cleaned
148 * before the driver removal callback is complete and therefore can't be
149 * deferred to a drmm action.
150 */
xe_gt_remove(struct xe_gt * gt)151 void xe_gt_remove(struct xe_gt *gt)
152 {
153 int i;
154
155 xe_uc_remove(>->uc);
156
157 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
158 xe_hw_fence_irq_finish(>->fence_irq[i]);
159
160 xe_gt_disable_host_l2_vram(gt);
161 }
162
163 static void gt_reset_worker(struct work_struct *w);
164
emit_nop_job(struct xe_gt * gt,struct xe_exec_queue * q)165 static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)
166 {
167 struct xe_sched_job *job;
168 struct xe_bb *bb;
169 struct dma_fence *fence;
170 long timeout;
171
172 bb = xe_bb_new(gt, 4, false);
173 if (IS_ERR(bb))
174 return PTR_ERR(bb);
175
176 job = xe_bb_create_job(q, bb);
177 if (IS_ERR(job)) {
178 xe_bb_free(bb, NULL);
179 return PTR_ERR(job);
180 }
181
182 xe_sched_job_arm(job);
183 fence = dma_fence_get(&job->drm.s_fence->finished);
184 xe_sched_job_push(job);
185
186 timeout = dma_fence_wait_timeout(fence, false, HZ);
187 dma_fence_put(fence);
188 xe_bb_free(bb, NULL);
189 if (timeout < 0)
190 return timeout;
191 else if (!timeout)
192 return -ETIME;
193
194 return 0;
195 }
196
197 /*
198 * Convert back from encoded value to type-safe, only to be used when reg.mcr
199 * is true
200 */
to_xe_reg_mcr(const struct xe_reg reg)201 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
202 {
203 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
204 }
205
emit_wa_job(struct xe_gt * gt,struct xe_exec_queue * q)206 static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)
207 {
208 struct xe_reg_sr *sr = &q->hwe->reg_lrc;
209 struct xe_reg_sr_entry *entry;
210 unsigned long idx;
211 struct xe_sched_job *job;
212 struct xe_bb *bb;
213 struct dma_fence *fence;
214 long timeout;
215 int count = 0;
216
217 if (q->hwe->class == XE_ENGINE_CLASS_RENDER)
218 /* Big enough to emit all of the context's 3DSTATE */
219 bb = xe_bb_new(gt, xe_gt_lrc_size(gt, q->hwe->class), false);
220 else
221 /* Just pick a large BB size */
222 bb = xe_bb_new(gt, SZ_4K, false);
223
224 if (IS_ERR(bb))
225 return PTR_ERR(bb);
226
227 xa_for_each(&sr->xa, idx, entry)
228 ++count;
229
230 if (count) {
231 xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name);
232
233 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);
234
235 xa_for_each(&sr->xa, idx, entry) {
236 struct xe_reg reg = entry->reg;
237 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
238 u32 val;
239
240 /*
241 * Skip reading the register if it's not really needed
242 */
243 if (reg.masked)
244 val = entry->clr_bits << 16;
245 else if (entry->clr_bits + 1)
246 val = (reg.mcr ?
247 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
248 xe_mmio_read32(gt, reg)) & (~entry->clr_bits);
249 else
250 val = 0;
251
252 val |= entry->set_bits;
253
254 bb->cs[bb->len++] = reg.addr;
255 bb->cs[bb->len++] = val;
256 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
257 }
258 }
259
260 xe_lrc_emit_hwe_state_instructions(q, bb);
261
262 job = xe_bb_create_job(q, bb);
263 if (IS_ERR(job)) {
264 xe_bb_free(bb, NULL);
265 return PTR_ERR(job);
266 }
267
268 xe_sched_job_arm(job);
269 fence = dma_fence_get(&job->drm.s_fence->finished);
270 xe_sched_job_push(job);
271
272 timeout = dma_fence_wait_timeout(fence, false, HZ);
273 dma_fence_put(fence);
274 xe_bb_free(bb, NULL);
275 if (timeout < 0)
276 return timeout;
277 else if (!timeout)
278 return -ETIME;
279
280 return 0;
281 }
282
xe_gt_record_default_lrcs(struct xe_gt * gt)283 int xe_gt_record_default_lrcs(struct xe_gt *gt)
284 {
285 struct xe_device *xe = gt_to_xe(gt);
286 struct xe_hw_engine *hwe;
287 enum xe_hw_engine_id id;
288 int err = 0;
289
290 for_each_hw_engine(hwe, gt, id) {
291 struct xe_exec_queue *q, *nop_q;
292 void *default_lrc;
293
294 if (gt->default_lrc[hwe->class])
295 continue;
296
297 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
298 xe_wa_process_lrc(hwe);
299 xe_hw_engine_setup_default_lrc_state(hwe);
300 xe_tuning_process_lrc(hwe);
301
302 default_lrc = drmm_kzalloc(&xe->drm,
303 xe_gt_lrc_size(gt, hwe->class),
304 GFP_KERNEL);
305 if (!default_lrc)
306 return -ENOMEM;
307
308 q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1,
309 hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
310 if (IS_ERR(q)) {
311 err = PTR_ERR(q);
312 xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n",
313 hwe->name, q);
314 return err;
315 }
316
317 /* Prime golden LRC with known good state */
318 err = emit_wa_job(gt, q);
319 if (err) {
320 xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n",
321 hwe->name, ERR_PTR(err), q->guc->id);
322 goto put_exec_queue;
323 }
324
325 nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance),
326 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
327 if (IS_ERR(nop_q)) {
328 err = PTR_ERR(nop_q);
329 xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n",
330 hwe->name, nop_q);
331 goto put_exec_queue;
332 }
333
334 /* Switch to different LRC */
335 err = emit_nop_job(gt, nop_q);
336 if (err) {
337 xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n",
338 hwe->name, ERR_PTR(err), nop_q->guc->id);
339 goto put_nop_q;
340 }
341
342 /* Reload golden LRC to record the effect of any indirect W/A */
343 err = emit_nop_job(gt, q);
344 if (err) {
345 xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n",
346 hwe->name, ERR_PTR(err), q->guc->id);
347 goto put_nop_q;
348 }
349
350 xe_map_memcpy_from(xe, default_lrc,
351 &q->lrc[0]->bo->vmap,
352 xe_lrc_pphwsp_offset(q->lrc[0]),
353 xe_gt_lrc_size(gt, hwe->class));
354
355 gt->default_lrc[hwe->class] = default_lrc;
356 put_nop_q:
357 xe_exec_queue_put(nop_q);
358 put_exec_queue:
359 xe_exec_queue_put(q);
360 if (err)
361 break;
362 }
363
364 return err;
365 }
366
xe_gt_init_early(struct xe_gt * gt)367 int xe_gt_init_early(struct xe_gt *gt)
368 {
369 int err;
370
371 if (IS_SRIOV_PF(gt_to_xe(gt))) {
372 err = xe_gt_sriov_pf_init_early(gt);
373 if (err)
374 return err;
375 }
376
377 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt));
378
379 err = xe_wa_init(gt);
380 if (err)
381 return err;
382
383 xe_wa_process_oob(gt);
384
385 xe_force_wake_init_gt(gt, gt_to_fw(gt));
386 spin_lock_init(>->global_invl_lock);
387
388 err = xe_gt_tlb_invalidation_init_early(gt);
389 if (err)
390 return err;
391
392 xe_mocs_init_early(gt);
393
394 return 0;
395 }
396
dump_pat_on_error(struct xe_gt * gt)397 static void dump_pat_on_error(struct xe_gt *gt)
398 {
399 struct drm_printer p;
400 char prefix[32];
401
402 snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id);
403 p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix);
404
405 xe_pat_dump(gt, &p);
406 }
407
gt_fw_domain_init(struct xe_gt * gt)408 static int gt_fw_domain_init(struct xe_gt *gt)
409 {
410 int err, i;
411
412 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
413 if (err)
414 goto err_hw_fence_irq;
415
416 if (!xe_gt_is_media_type(gt)) {
417 err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
418 if (err)
419 goto err_force_wake;
420 if (IS_SRIOV_PF(gt_to_xe(gt)))
421 xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt);
422 }
423
424 /* Enable per hw engine IRQs */
425 xe_irq_enable_hwe(gt);
426
427 /* Rerun MCR init as we now have hw engine list */
428 xe_gt_mcr_init(gt);
429
430 err = xe_hw_engines_init_early(gt);
431 if (err)
432 goto err_force_wake;
433
434 err = xe_hw_engine_class_sysfs_init(gt);
435 if (err)
436 goto err_force_wake;
437
438 /* Initialize CCS mode sysfs after early initialization of HW engines */
439 err = xe_gt_ccs_mode_sysfs_init(gt);
440 if (err)
441 goto err_force_wake;
442
443 /*
444 * Stash hardware-reported version. Since this register does not exist
445 * on pre-MTL platforms, reading it there will (correctly) return 0.
446 */
447 gt->info.gmdid = xe_mmio_read32(gt, GMD_ID);
448
449 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
450 XE_WARN_ON(err);
451
452 return 0;
453
454 err_force_wake:
455 dump_pat_on_error(gt);
456 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
457 err_hw_fence_irq:
458 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
459 xe_hw_fence_irq_finish(>->fence_irq[i]);
460
461 return err;
462 }
463
all_fw_domain_init(struct xe_gt * gt)464 static int all_fw_domain_init(struct xe_gt *gt)
465 {
466 int err, i;
467
468 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
469 if (err)
470 goto err_hw_fence_irq;
471
472 xe_gt_mcr_set_implicit_defaults(gt);
473 xe_wa_process_gt(gt);
474 xe_tuning_process_gt(gt);
475 xe_reg_sr_apply_mmio(>->reg_sr, gt);
476
477 err = xe_gt_clock_init(gt);
478 if (err)
479 goto err_force_wake;
480
481 xe_mocs_init(gt);
482 err = xe_execlist_init(gt);
483 if (err)
484 goto err_force_wake;
485
486 err = xe_hw_engines_init(gt);
487 if (err)
488 goto err_force_wake;
489
490 err = xe_uc_init_post_hwconfig(>->uc);
491 if (err)
492 goto err_force_wake;
493
494 if (!xe_gt_is_media_type(gt)) {
495 /*
496 * USM has its only SA pool to non-block behind user operations
497 */
498 if (gt_to_xe(gt)->info.has_usm) {
499 struct xe_device *xe = gt_to_xe(gt);
500
501 gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
502 IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
503 if (IS_ERR(gt->usm.bb_pool)) {
504 err = PTR_ERR(gt->usm.bb_pool);
505 goto err_force_wake;
506 }
507 }
508 }
509
510 if (!xe_gt_is_media_type(gt)) {
511 struct xe_tile *tile = gt_to_tile(gt);
512
513 tile->migrate = xe_migrate_init(tile);
514 if (IS_ERR(tile->migrate)) {
515 err = PTR_ERR(tile->migrate);
516 goto err_force_wake;
517 }
518 }
519
520 err = xe_uc_init_hw(>->uc);
521 if (err)
522 goto err_force_wake;
523
524 /* Configure default CCS mode of 1 engine with all resources */
525 if (xe_gt_ccs_mode_enabled(gt)) {
526 gt->ccs_mode = 1;
527 xe_gt_apply_ccs_mode(gt);
528 }
529
530 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))
531 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
532
533 if (IS_SRIOV_PF(gt_to_xe(gt)))
534 xe_gt_sriov_pf_init_hw(gt);
535
536 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
537 XE_WARN_ON(err);
538
539 return 0;
540
541 err_force_wake:
542 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
543 err_hw_fence_irq:
544 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
545 xe_hw_fence_irq_finish(>->fence_irq[i]);
546
547 return err;
548 }
549
550 /*
551 * Initialize enough GT to be able to load GuC in order to obtain hwconfig and
552 * enable CTB communication.
553 */
xe_gt_init_hwconfig(struct xe_gt * gt)554 int xe_gt_init_hwconfig(struct xe_gt *gt)
555 {
556 int err;
557
558 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
559 if (err)
560 goto out;
561
562 xe_gt_mcr_init_early(gt);
563 xe_pat_init(gt);
564
565 err = xe_uc_init(>->uc);
566 if (err)
567 goto out_fw;
568
569 err = xe_uc_init_hwconfig(>->uc);
570 if (err)
571 goto out_fw;
572
573 xe_gt_topology_init(gt);
574 xe_gt_mcr_init(gt);
575 xe_gt_enable_host_l2_vram(gt);
576
577 out_fw:
578 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
579 out:
580 return err;
581 }
582
xe_gt_init(struct xe_gt * gt)583 int xe_gt_init(struct xe_gt *gt)
584 {
585 int err;
586 int i;
587
588 INIT_WORK(>->reset.worker, gt_reset_worker);
589
590 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {
591 gt->ring_ops[i] = xe_ring_ops_get(gt, i);
592 xe_hw_fence_irq_init(>->fence_irq[i]);
593 }
594
595 err = xe_gt_sysfs_init(gt);
596 if (err)
597 return err;
598
599 err = gt_fw_domain_init(gt);
600 if (err)
601 return err;
602
603 err = xe_gt_pagefault_init(gt);
604 if (err)
605 return err;
606
607 err = xe_gt_idle_init(>->gtidle);
608 if (err)
609 return err;
610
611 err = xe_gt_freq_init(gt);
612 if (err)
613 return err;
614
615 xe_force_wake_init_engines(gt, gt_to_fw(gt));
616
617 err = all_fw_domain_init(gt);
618 if (err)
619 return err;
620
621 xe_gt_record_user_engines(gt);
622
623 return 0;
624 }
625
xe_gt_record_user_engines(struct xe_gt * gt)626 void xe_gt_record_user_engines(struct xe_gt *gt)
627 {
628 struct xe_hw_engine *hwe;
629 enum xe_hw_engine_id id;
630
631 gt->user_engines.mask = 0;
632 memset(gt->user_engines.instances_per_class, 0,
633 sizeof(gt->user_engines.instances_per_class));
634
635 for_each_hw_engine(hwe, gt, id) {
636 if (xe_hw_engine_is_reserved(hwe))
637 continue;
638
639 gt->user_engines.mask |= BIT_ULL(id);
640 gt->user_engines.instances_per_class[hwe->class]++;
641 }
642
643 xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask)
644 == gt->info.engine_mask);
645 }
646
do_gt_reset(struct xe_gt * gt)647 static int do_gt_reset(struct xe_gt *gt)
648 {
649 int err;
650
651 if (IS_SRIOV_VF(gt_to_xe(gt)))
652 return xe_gt_sriov_vf_reset(gt);
653
654 xe_gsc_wa_14015076503(gt, true);
655
656 xe_mmio_write32(gt, GDRST, GRDOM_FULL);
657 err = xe_mmio_wait32(gt, GDRST, GRDOM_FULL, 0, 5000, NULL, false);
658 if (err)
659 xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",
660 ERR_PTR(err));
661
662 xe_gsc_wa_14015076503(gt, false);
663
664 return err;
665 }
666
vf_gt_restart(struct xe_gt * gt)667 static int vf_gt_restart(struct xe_gt *gt)
668 {
669 int err;
670
671 err = xe_uc_sanitize_reset(>->uc);
672 if (err)
673 return err;
674
675 err = xe_uc_init_hw(>->uc);
676 if (err)
677 return err;
678
679 err = xe_uc_start(>->uc);
680 if (err)
681 return err;
682
683 return 0;
684 }
685
do_gt_restart(struct xe_gt * gt)686 static int do_gt_restart(struct xe_gt *gt)
687 {
688 struct xe_hw_engine *hwe;
689 enum xe_hw_engine_id id;
690 int err;
691
692 if (IS_SRIOV_VF(gt_to_xe(gt)))
693 return vf_gt_restart(gt);
694
695 xe_pat_init(gt);
696
697 xe_gt_enable_host_l2_vram(gt);
698
699 xe_gt_mcr_set_implicit_defaults(gt);
700 xe_reg_sr_apply_mmio(>->reg_sr, gt);
701
702 err = xe_wopcm_init(>->uc.wopcm);
703 if (err)
704 return err;
705
706 for_each_hw_engine(hwe, gt, id)
707 xe_hw_engine_enable_ring(hwe);
708
709 err = xe_uc_sanitize_reset(>->uc);
710 if (err)
711 return err;
712
713 err = xe_uc_init_hw(>->uc);
714 if (err)
715 return err;
716
717 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))
718 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
719
720 if (IS_SRIOV_PF(gt_to_xe(gt)))
721 xe_gt_sriov_pf_init_hw(gt);
722
723 xe_mocs_init(gt);
724 err = xe_uc_start(>->uc);
725 if (err)
726 return err;
727
728 for_each_hw_engine(hwe, gt, id) {
729 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
730 xe_reg_sr_apply_whitelist(hwe);
731 }
732
733 /* Get CCS mode in sync between sw/hw */
734 xe_gt_apply_ccs_mode(gt);
735
736 /* Restore GT freq to expected values */
737 xe_gt_sanitize_freq(gt);
738
739 if (IS_SRIOV_PF(gt_to_xe(gt)))
740 xe_gt_sriov_pf_restart(gt);
741
742 return 0;
743 }
744
gt_reset(struct xe_gt * gt)745 static int gt_reset(struct xe_gt *gt)
746 {
747 int err;
748
749 if (xe_device_wedged(gt_to_xe(gt)))
750 return -ECANCELED;
751
752 /* We only support GT resets with GuC submission */
753 if (!xe_device_uc_enabled(gt_to_xe(gt)))
754 return -ENODEV;
755
756 xe_gt_info(gt, "reset started\n");
757
758 xe_pm_runtime_get(gt_to_xe(gt));
759
760 if (xe_fault_inject_gt_reset()) {
761 err = -ECANCELED;
762 goto err_fail;
763 }
764
765 xe_gt_sanitize(gt);
766
767 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
768 if (err)
769 goto err_msg;
770
771 if (IS_SRIOV_PF(gt_to_xe(gt)))
772 xe_gt_sriov_pf_stop_prepare(gt);
773
774 xe_uc_gucrc_disable(>->uc);
775 xe_uc_stop_prepare(>->uc);
776 xe_gt_pagefault_reset(gt);
777
778 xe_uc_stop(>->uc);
779
780 xe_gt_tlb_invalidation_reset(gt);
781
782 err = do_gt_reset(gt);
783 if (err)
784 goto err_out;
785
786 err = do_gt_restart(gt);
787 if (err)
788 goto err_out;
789
790 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
791 XE_WARN_ON(err);
792 xe_pm_runtime_put(gt_to_xe(gt));
793
794 xe_gt_info(gt, "reset done\n");
795
796 return 0;
797
798 err_out:
799 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
800 err_msg:
801 XE_WARN_ON(xe_uc_start(>->uc));
802 err_fail:
803 xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
804
805 xe_device_declare_wedged(gt_to_xe(gt));
806 xe_pm_runtime_put(gt_to_xe(gt));
807
808 return err;
809 }
810
gt_reset_worker(struct work_struct * w)811 static void gt_reset_worker(struct work_struct *w)
812 {
813 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);
814
815 gt_reset(gt);
816 }
817
xe_gt_reset_async(struct xe_gt * gt)818 void xe_gt_reset_async(struct xe_gt *gt)
819 {
820 xe_gt_info(gt, "trying reset\n");
821
822 /* Don't do a reset while one is already in flight */
823 if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc))
824 return;
825
826 xe_gt_info(gt, "reset queued\n");
827 queue_work(gt->ordered_wq, >->reset.worker);
828 }
829
xe_gt_suspend_prepare(struct xe_gt * gt)830 void xe_gt_suspend_prepare(struct xe_gt *gt)
831 {
832 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
833
834 xe_uc_suspend_prepare(>->uc);
835
836 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
837 }
838
xe_gt_suspend(struct xe_gt * gt)839 int xe_gt_suspend(struct xe_gt *gt)
840 {
841 int err;
842
843 xe_gt_dbg(gt, "suspending\n");
844 xe_gt_sanitize(gt);
845
846 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
847 if (err)
848 goto err_msg;
849
850 err = xe_uc_suspend(>->uc);
851 if (err)
852 goto err_force_wake;
853
854 xe_gt_idle_disable_pg(gt);
855
856 xe_gt_disable_host_l2_vram(gt);
857
858 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
859 xe_gt_dbg(gt, "suspended\n");
860
861 return 0;
862
863 err_force_wake:
864 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
865 err_msg:
866 xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
867
868 return err;
869 }
870
xe_gt_shutdown(struct xe_gt * gt)871 void xe_gt_shutdown(struct xe_gt *gt)
872 {
873 xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
874 do_gt_reset(gt);
875 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
876 }
877
878 /**
879 * xe_gt_sanitize_freq() - Restore saved frequencies if necessary.
880 * @gt: the GT object
881 *
882 * Called after driver init/GSC load completes to restore GT frequencies if we
883 * limited them for any WAs.
884 */
xe_gt_sanitize_freq(struct xe_gt * gt)885 int xe_gt_sanitize_freq(struct xe_gt *gt)
886 {
887 int ret = 0;
888
889 if ((!xe_uc_fw_is_available(>->uc.gsc.fw) ||
890 xe_uc_fw_is_loaded(>->uc.gsc.fw) ||
891 xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) &&
892 XE_WA(gt, 22019338487))
893 ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc);
894
895 return ret;
896 }
897
xe_gt_resume(struct xe_gt * gt)898 int xe_gt_resume(struct xe_gt *gt)
899 {
900 int err;
901
902 xe_gt_dbg(gt, "resuming\n");
903 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
904 if (err)
905 goto err_msg;
906
907 err = do_gt_restart(gt);
908 if (err)
909 goto err_force_wake;
910
911 xe_gt_idle_enable_pg(gt);
912
913 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
914 xe_gt_dbg(gt, "resumed\n");
915
916 return 0;
917
918 err_force_wake:
919 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
920 err_msg:
921 xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
922
923 return err;
924 }
925
xe_gt_hw_engine(struct xe_gt * gt,enum xe_engine_class class,u16 instance,bool logical)926 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
927 enum xe_engine_class class,
928 u16 instance, bool logical)
929 {
930 struct xe_hw_engine *hwe;
931 enum xe_hw_engine_id id;
932
933 for_each_hw_engine(hwe, gt, id)
934 if (hwe->class == class &&
935 ((!logical && hwe->instance == instance) ||
936 (logical && hwe->logical_instance == instance)))
937 return hwe;
938
939 return NULL;
940 }
941
xe_gt_any_hw_engine_by_reset_domain(struct xe_gt * gt,enum xe_engine_class class)942 struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,
943 enum xe_engine_class class)
944 {
945 struct xe_hw_engine *hwe;
946 enum xe_hw_engine_id id;
947
948 for_each_hw_engine(hwe, gt, id) {
949 switch (class) {
950 case XE_ENGINE_CLASS_RENDER:
951 case XE_ENGINE_CLASS_COMPUTE:
952 if (hwe->class == XE_ENGINE_CLASS_RENDER ||
953 hwe->class == XE_ENGINE_CLASS_COMPUTE)
954 return hwe;
955 break;
956 default:
957 if (hwe->class == class)
958 return hwe;
959 }
960 }
961
962 return NULL;
963 }
964
xe_gt_any_hw_engine(struct xe_gt * gt)965 struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt)
966 {
967 struct xe_hw_engine *hwe;
968 enum xe_hw_engine_id id;
969
970 for_each_hw_engine(hwe, gt, id)
971 return hwe;
972
973 return NULL;
974 }
975
976 /**
977 * xe_gt_declare_wedged() - Declare GT wedged
978 * @gt: the GT object
979 *
980 * Wedge the GT which stops all submission, saves desired debug state, and
981 * cleans up anything which could timeout.
982 */
xe_gt_declare_wedged(struct xe_gt * gt)983 void xe_gt_declare_wedged(struct xe_gt *gt)
984 {
985 xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);
986
987 xe_uc_declare_wedged(>->uc);
988 xe_gt_tlb_invalidation_reset(gt);
989 }
990