1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_ct.h"
7
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 #include <linux/delay.h>
11
12 #include <kunit/static_stub.h>
13
14 #include <drm/drm_managed.h>
15
16 #include "abi/guc_actions_abi.h"
17 #include "abi/guc_actions_sriov_abi.h"
18 #include "abi/guc_klvs_abi.h"
19 #include "xe_bo.h"
20 #include "xe_device.h"
21 #include "xe_gt.h"
22 #include "xe_gt_pagefault.h"
23 #include "xe_gt_printk.h"
24 #include "xe_gt_sriov_pf_control.h"
25 #include "xe_gt_sriov_pf_monitor.h"
26 #include "xe_gt_tlb_invalidation.h"
27 #include "xe_guc.h"
28 #include "xe_guc_log.h"
29 #include "xe_guc_relay.h"
30 #include "xe_guc_submit.h"
31 #include "xe_map.h"
32 #include "xe_pm.h"
33 #include "xe_trace_guc.h"
34
35 static void receive_g2h(struct xe_guc_ct *ct);
36 static void g2h_worker_func(struct work_struct *w);
37 static void safe_mode_worker_func(struct work_struct *w);
38 static void ct_exit_safe_mode(struct xe_guc_ct *ct);
39
40 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
41 enum {
42 /* Internal states, not error conditions */
43 CT_DEAD_STATE_REARM, /* 0x0001 */
44 CT_DEAD_STATE_CAPTURE, /* 0x0002 */
45
46 /* Error conditions */
47 CT_DEAD_SETUP, /* 0x0004 */
48 CT_DEAD_H2G_WRITE, /* 0x0008 */
49 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */
50 CT_DEAD_G2H_READ, /* 0x0020 */
51 CT_DEAD_G2H_RECV, /* 0x0040 */
52 CT_DEAD_G2H_RELEASE, /* 0x0080 */
53 CT_DEAD_DEADLOCK, /* 0x0100 */
54 CT_DEAD_PROCESS_FAILED, /* 0x0200 */
55 CT_DEAD_FAST_G2H, /* 0x0400 */
56 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */
57 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */
58 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */
59 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */
60 };
61
62 static void ct_dead_worker_func(struct work_struct *w);
63 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code);
64
65 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
66 #else
67 #define CT_DEAD(ct, ctb, reason) \
68 do { \
69 struct guc_ctb *_ctb = (ctb); \
70 if (_ctb) \
71 _ctb->info.broken = true; \
72 } while (0)
73 #endif
74
75 /* Used when a CT send wants to block and / or receive data */
76 struct g2h_fence {
77 u32 *response_buffer;
78 u32 seqno;
79 u32 response_data;
80 u16 response_len;
81 u16 error;
82 u16 hint;
83 u16 reason;
84 bool retry;
85 bool fail;
86 bool done;
87 };
88
g2h_fence_init(struct g2h_fence * g2h_fence,u32 * response_buffer)89 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
90 {
91 g2h_fence->response_buffer = response_buffer;
92 g2h_fence->response_data = 0;
93 g2h_fence->response_len = 0;
94 g2h_fence->fail = false;
95 g2h_fence->retry = false;
96 g2h_fence->done = false;
97 g2h_fence->seqno = ~0x0;
98 }
99
g2h_fence_needs_alloc(struct g2h_fence * g2h_fence)100 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
101 {
102 return g2h_fence->seqno == ~0x0;
103 }
104
105 static struct xe_guc *
ct_to_guc(struct xe_guc_ct * ct)106 ct_to_guc(struct xe_guc_ct *ct)
107 {
108 return container_of(ct, struct xe_guc, ct);
109 }
110
111 static struct xe_gt *
ct_to_gt(struct xe_guc_ct * ct)112 ct_to_gt(struct xe_guc_ct *ct)
113 {
114 return container_of(ct, struct xe_gt, uc.guc.ct);
115 }
116
117 static struct xe_device *
ct_to_xe(struct xe_guc_ct * ct)118 ct_to_xe(struct xe_guc_ct *ct)
119 {
120 return gt_to_xe(ct_to_gt(ct));
121 }
122
123 /**
124 * DOC: GuC CTB Blob
125 *
126 * We allocate single blob to hold both CTB descriptors and buffers:
127 *
128 * +--------+-----------------------------------------------+------+
129 * | offset | contents | size |
130 * +========+===============================================+======+
131 * | 0x0000 | H2G CTB Descriptor (send) | |
132 * +--------+-----------------------------------------------+ 4K |
133 * | 0x0800 | G2H CTB Descriptor (g2h) | |
134 * +--------+-----------------------------------------------+------+
135 * | 0x1000 | H2G CT Buffer (send) | n*4K |
136 * | | | |
137 * +--------+-----------------------------------------------+------+
138 * | 0x1000 | G2H CT Buffer (g2h) | m*4K |
139 * | + n*4K | | |
140 * +--------+-----------------------------------------------+------+
141 *
142 * Size of each ``CT Buffer`` must be multiple of 4K.
143 * We don't expect too many messages in flight at any time, unless we are
144 * using the GuC submission. In that case each request requires a minimum
145 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
146 * enough space to avoid backpressure on the driver. We increase the size
147 * of the receive buffer (relative to the send) to ensure a G2H response
148 * CTB has a landing spot.
149 *
150 * In addition to submissions, the G2H buffer needs to be able to hold
151 * enough space for recoverable page fault notifications. The number of
152 * page faults is interrupt driven and can be as much as the number of
153 * compute resources available. However, most of the actual work for these
154 * is in a separate page fault worker thread. Therefore we only need to
155 * make sure the queue has enough space to handle all of the submissions
156 * and responses and an extra buffer for incoming page faults.
157 */
158
159 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
160 #define CTB_H2G_BUFFER_SIZE (SZ_4K)
161 #define CTB_G2H_BUFFER_SIZE (SZ_128K)
162 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)
163
164 /**
165 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
166 * CT command queue
167 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.
168 *
169 * Observation is that a 4KiB buffer full of commands takes a little over a
170 * second to process. Use that to calculate maximum time to process a full CT
171 * command queue.
172 *
173 * Return: Maximum time to process a full CT queue in jiffies.
174 */
xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct * ct)175 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
176 {
177 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
178 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
179 }
180
guc_ct_size(void)181 static size_t guc_ct_size(void)
182 {
183 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +
184 CTB_G2H_BUFFER_SIZE;
185 }
186
guc_ct_fini(struct drm_device * drm,void * arg)187 static void guc_ct_fini(struct drm_device *drm, void *arg)
188 {
189 struct xe_guc_ct *ct = arg;
190
191 ct_exit_safe_mode(ct);
192 destroy_workqueue(ct->g2h_wq);
193 xa_destroy(&ct->fence_lookup);
194 }
195
primelockdep(struct xe_guc_ct * ct)196 static void primelockdep(struct xe_guc_ct *ct)
197 {
198 if (!IS_ENABLED(CONFIG_LOCKDEP))
199 return;
200
201 fs_reclaim_acquire(GFP_KERNEL);
202 might_lock(&ct->lock);
203 fs_reclaim_release(GFP_KERNEL);
204 }
205
xe_guc_ct_init(struct xe_guc_ct * ct)206 int xe_guc_ct_init(struct xe_guc_ct *ct)
207 {
208 struct xe_device *xe = ct_to_xe(ct);
209 struct xe_gt *gt = ct_to_gt(ct);
210 struct xe_tile *tile = gt_to_tile(gt);
211 struct xe_bo *bo;
212 int err;
213
214 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
215
216 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0);
217 if (!ct->g2h_wq)
218 return -ENOMEM;
219
220 spin_lock_init(&ct->fast_lock);
221 xa_init(&ct->fence_lookup);
222 INIT_WORK(&ct->g2h_worker, g2h_worker_func);
223 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
224 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
225 spin_lock_init(&ct->dead.lock);
226 INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
227 #endif
228 init_waitqueue_head(&ct->wq);
229 init_waitqueue_head(&ct->g2h_fence_wq);
230
231 err = drmm_mutex_init(&xe->drm, &ct->lock);
232 if (err)
233 return err;
234
235 primelockdep(ct);
236
237 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
238 XE_BO_FLAG_SYSTEM |
239 XE_BO_FLAG_GGTT |
240 XE_BO_FLAG_GGTT_INVALIDATE);
241 if (IS_ERR(bo))
242 return PTR_ERR(bo);
243
244 ct->bo = bo;
245
246 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
247 if (err)
248 return err;
249
250 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
251 ct->state = XE_GUC_CT_STATE_DISABLED;
252 return 0;
253 }
254
255 #define desc_read(xe_, guc_ctb__, field_) \
256 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \
257 struct guc_ct_buffer_desc, field_)
258
259 #define desc_write(xe_, guc_ctb__, field_, val_) \
260 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \
261 struct guc_ct_buffer_desc, field_, val_)
262
guc_ct_ctb_h2g_init(struct xe_device * xe,struct guc_ctb * h2g,struct iosys_map * map)263 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
264 struct iosys_map *map)
265 {
266 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
267 h2g->info.resv_space = 0;
268 h2g->info.tail = 0;
269 h2g->info.head = 0;
270 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
271 h2g->info.size) -
272 h2g->info.resv_space;
273 h2g->info.broken = false;
274
275 h2g->desc = *map;
276 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
277
278 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);
279 }
280
guc_ct_ctb_g2h_init(struct xe_device * xe,struct guc_ctb * g2h,struct iosys_map * map)281 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
282 struct iosys_map *map)
283 {
284 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
285 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
286 g2h->info.head = 0;
287 g2h->info.tail = 0;
288 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
289 g2h->info.size) -
290 g2h->info.resv_space;
291 g2h->info.broken = false;
292
293 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
294 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
295
296 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +
297 CTB_H2G_BUFFER_SIZE);
298 }
299
guc_ct_ctb_h2g_register(struct xe_guc_ct * ct)300 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
301 {
302 struct xe_guc *guc = ct_to_guc(ct);
303 u32 desc_addr, ctb_addr, size;
304 int err;
305
306 desc_addr = xe_bo_ggtt_addr(ct->bo);
307 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;
308 size = ct->ctbs.h2g.info.size * sizeof(u32);
309
310 err = xe_guc_self_cfg64(guc,
311 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
312 desc_addr);
313 if (err)
314 return err;
315
316 err = xe_guc_self_cfg64(guc,
317 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
318 ctb_addr);
319 if (err)
320 return err;
321
322 return xe_guc_self_cfg32(guc,
323 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
324 size);
325 }
326
guc_ct_ctb_g2h_register(struct xe_guc_ct * ct)327 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
328 {
329 struct xe_guc *guc = ct_to_guc(ct);
330 u32 desc_addr, ctb_addr, size;
331 int err;
332
333 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
334 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +
335 CTB_H2G_BUFFER_SIZE;
336 size = ct->ctbs.g2h.info.size * sizeof(u32);
337
338 err = xe_guc_self_cfg64(guc,
339 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
340 desc_addr);
341 if (err)
342 return err;
343
344 err = xe_guc_self_cfg64(guc,
345 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
346 ctb_addr);
347 if (err)
348 return err;
349
350 return xe_guc_self_cfg32(guc,
351 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
352 size);
353 }
354
guc_ct_control_toggle(struct xe_guc_ct * ct,bool enable)355 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
356 {
357 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
358 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
359 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
360 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
361 GUC_ACTION_HOST2GUC_CONTROL_CTB),
362 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
363 enable ? GUC_CTB_CONTROL_ENABLE :
364 GUC_CTB_CONTROL_DISABLE),
365 };
366 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
367
368 return ret > 0 ? -EPROTO : ret;
369 }
370
xe_guc_ct_set_state(struct xe_guc_ct * ct,enum xe_guc_ct_state state)371 static void xe_guc_ct_set_state(struct xe_guc_ct *ct,
372 enum xe_guc_ct_state state)
373 {
374 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */
375 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */
376
377 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
378 state == XE_GUC_CT_STATE_STOPPED);
379
380 if (ct->g2h_outstanding)
381 xe_pm_runtime_put(ct_to_xe(ct));
382 ct->g2h_outstanding = 0;
383 ct->state = state;
384
385 spin_unlock_irq(&ct->fast_lock);
386
387 /*
388 * Lockdep doesn't like this under the fast lock and he destroy only
389 * needs to be serialized with the send path which ct lock provides.
390 */
391 xa_destroy(&ct->fence_lookup);
392
393 mutex_unlock(&ct->lock);
394 }
395
ct_needs_safe_mode(struct xe_guc_ct * ct)396 static bool ct_needs_safe_mode(struct xe_guc_ct *ct)
397 {
398 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));
399 }
400
ct_restart_safe_mode_worker(struct xe_guc_ct * ct)401 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)
402 {
403 if (!ct_needs_safe_mode(ct))
404 return false;
405
406 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);
407 return true;
408 }
409
safe_mode_worker_func(struct work_struct * w)410 static void safe_mode_worker_func(struct work_struct *w)
411 {
412 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);
413
414 receive_g2h(ct);
415
416 if (!ct_restart_safe_mode_worker(ct))
417 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");
418 }
419
ct_enter_safe_mode(struct xe_guc_ct * ct)420 static void ct_enter_safe_mode(struct xe_guc_ct *ct)
421 {
422 if (ct_restart_safe_mode_worker(ct))
423 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");
424 }
425
ct_exit_safe_mode(struct xe_guc_ct * ct)426 static void ct_exit_safe_mode(struct xe_guc_ct *ct)
427 {
428 if (cancel_delayed_work_sync(&ct->safe_mode_worker))
429 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");
430 }
431
xe_guc_ct_enable(struct xe_guc_ct * ct)432 int xe_guc_ct_enable(struct xe_guc_ct *ct)
433 {
434 struct xe_device *xe = ct_to_xe(ct);
435 struct xe_gt *gt = ct_to_gt(ct);
436 int err;
437
438 xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
439
440 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
441 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
442
443 err = guc_ct_ctb_h2g_register(ct);
444 if (err)
445 goto err_out;
446
447 err = guc_ct_ctb_g2h_register(ct);
448 if (err)
449 goto err_out;
450
451 err = guc_ct_control_toggle(ct, true);
452 if (err)
453 goto err_out;
454
455 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED);
456
457 smp_mb();
458 wake_up_all(&ct->wq);
459 xe_gt_dbg(gt, "GuC CT communication channel enabled\n");
460
461 if (ct_needs_safe_mode(ct))
462 ct_enter_safe_mode(ct);
463
464 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
465 /*
466 * The CT has now been reset so the dumper can be re-armed
467 * after any existing dead state has been dumped.
468 */
469 spin_lock_irq(&ct->dead.lock);
470 if (ct->dead.reason)
471 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM);
472 spin_unlock_irq(&ct->dead.lock);
473 #endif
474
475 return 0;
476
477 err_out:
478 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
479 CT_DEAD(ct, NULL, SETUP);
480
481 return err;
482 }
483
stop_g2h_handler(struct xe_guc_ct * ct)484 static void stop_g2h_handler(struct xe_guc_ct *ct)
485 {
486 cancel_work_sync(&ct->g2h_worker);
487 }
488
489 /**
490 * xe_guc_ct_disable - Set GuC to disabled state
491 * @ct: the &xe_guc_ct
492 *
493 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected
494 * in this transition.
495 */
xe_guc_ct_disable(struct xe_guc_ct * ct)496 void xe_guc_ct_disable(struct xe_guc_ct *ct)
497 {
498 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED);
499 ct_exit_safe_mode(ct);
500 stop_g2h_handler(ct);
501 }
502
503 /**
504 * xe_guc_ct_stop - Set GuC to stopped state
505 * @ct: the &xe_guc_ct
506 *
507 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h
508 */
xe_guc_ct_stop(struct xe_guc_ct * ct)509 void xe_guc_ct_stop(struct xe_guc_ct *ct)
510 {
511 if (!xe_guc_ct_initialized(ct))
512 return;
513
514 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED);
515 stop_g2h_handler(ct);
516 }
517
h2g_has_room(struct xe_guc_ct * ct,u32 cmd_len)518 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
519 {
520 struct guc_ctb *h2g = &ct->ctbs.h2g;
521
522 lockdep_assert_held(&ct->lock);
523
524 if (cmd_len > h2g->info.space) {
525 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
526
527 if (h2g->info.head > h2g->info.size) {
528 struct xe_device *xe = ct_to_xe(ct);
529 u32 desc_status = desc_read(xe, h2g, status);
530
531 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
532
533 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n",
534 h2g->info.head, h2g->info.size);
535 CT_DEAD(ct, h2g, H2G_HAS_ROOM);
536 return false;
537 }
538
539 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
540 h2g->info.size) -
541 h2g->info.resv_space;
542 if (cmd_len > h2g->info.space)
543 return false;
544 }
545
546 return true;
547 }
548
g2h_has_room(struct xe_guc_ct * ct,u32 g2h_len)549 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
550 {
551 if (!g2h_len)
552 return true;
553
554 lockdep_assert_held(&ct->fast_lock);
555
556 return ct->ctbs.g2h.info.space > g2h_len;
557 }
558
has_room(struct xe_guc_ct * ct,u32 cmd_len,u32 g2h_len)559 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
560 {
561 lockdep_assert_held(&ct->lock);
562
563 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
564 return -EBUSY;
565
566 return 0;
567 }
568
h2g_reserve_space(struct xe_guc_ct * ct,u32 cmd_len)569 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
570 {
571 lockdep_assert_held(&ct->lock);
572 ct->ctbs.h2g.info.space -= cmd_len;
573 }
574
__g2h_reserve_space(struct xe_guc_ct * ct,u32 g2h_len,u32 num_g2h)575 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
576 {
577 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
578 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
579 (g2h_len && num_g2h));
580
581 if (g2h_len) {
582 lockdep_assert_held(&ct->fast_lock);
583
584 if (!ct->g2h_outstanding)
585 xe_pm_runtime_get_noresume(ct_to_xe(ct));
586
587 ct->ctbs.g2h.info.space -= g2h_len;
588 ct->g2h_outstanding += num_g2h;
589 }
590 }
591
__g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)592 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
593 {
594 bool bad = false;
595
596 lockdep_assert_held(&ct->fast_lock);
597
598 bad = ct->ctbs.g2h.info.space + g2h_len >
599 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space;
600 bad |= !ct->g2h_outstanding;
601
602 if (bad) {
603 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n",
604 ct->ctbs.g2h.info.space, g2h_len,
605 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space,
606 ct->ctbs.g2h.info.space + g2h_len,
607 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space,
608 ct->g2h_outstanding);
609 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE);
610 return;
611 }
612
613 ct->ctbs.g2h.info.space += g2h_len;
614 if (!--ct->g2h_outstanding)
615 xe_pm_runtime_put(ct_to_xe(ct));
616 }
617
g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)618 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
619 {
620 spin_lock_irq(&ct->fast_lock);
621 __g2h_release_space(ct, g2h_len);
622 spin_unlock_irq(&ct->fast_lock);
623 }
624
625 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
626
h2g_write(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 ct_fence_value,bool want_response)627 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
628 u32 ct_fence_value, bool want_response)
629 {
630 struct xe_device *xe = ct_to_xe(ct);
631 struct xe_gt *gt = ct_to_gt(ct);
632 struct guc_ctb *h2g = &ct->ctbs.h2g;
633 u32 cmd[H2G_CT_HEADERS];
634 u32 tail = h2g->info.tail;
635 u32 full_len;
636 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
637 tail * sizeof(u32));
638 u32 desc_status;
639
640 full_len = len + GUC_CTB_HDR_LEN;
641
642 lockdep_assert_held(&ct->lock);
643 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
644
645 desc_status = desc_read(xe, h2g, status);
646 if (desc_status) {
647 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
648 goto corrupted;
649 }
650
651 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
652 u32 desc_tail = desc_read(xe, h2g, tail);
653 u32 desc_head = desc_read(xe, h2g, head);
654
655 if (tail != desc_tail) {
656 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
657 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail);
658 goto corrupted;
659 }
660
661 if (tail > h2g->info.size) {
662 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
663 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n",
664 tail, h2g->info.size);
665 goto corrupted;
666 }
667
668 if (desc_head >= h2g->info.size) {
669 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
670 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n",
671 desc_head, h2g->info.size);
672 goto corrupted;
673 }
674 }
675
676 /* Command will wrap, zero fill (NOPs), return and check credits again */
677 if (tail + full_len > h2g->info.size) {
678 xe_map_memset(xe, &map, 0, 0,
679 (h2g->info.size - tail) * sizeof(u32));
680 h2g_reserve_space(ct, (h2g->info.size - tail));
681 h2g->info.tail = 0;
682 desc_write(xe, h2g, tail, h2g->info.tail);
683
684 return -EAGAIN;
685 }
686
687 /*
688 * dw0: CT header (including fence)
689 * dw1: HXG header (including action code)
690 * dw2+: action data
691 */
692 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
693 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
694 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
695 if (want_response) {
696 cmd[1] =
697 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
698 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
699 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
700 } else {
701 cmd[1] =
702 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) |
703 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
704 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
705 }
706
707 /* H2G header in cmd[1] replaces action[0] so: */
708 --len;
709 ++action;
710
711 /* Write H2G ensuring visable before descriptor update */
712 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
713 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
714 xe_device_wmb(xe);
715
716 /* Update local copies */
717 h2g->info.tail = (tail + full_len) % h2g->info.size;
718 h2g_reserve_space(ct, full_len);
719
720 /* Update descriptor */
721 desc_write(xe, h2g, tail, h2g->info.tail);
722
723 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
724 desc_read(xe, h2g, head), h2g->info.tail);
725
726 return 0;
727
728 corrupted:
729 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE);
730 return -EPIPE;
731 }
732
733 /*
734 * The CT protocol accepts a 16 bits fence. This field is fully owned by the
735 * driver, the GuC will just copy it to the reply message. Since we need to
736 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,
737 * we use one bit of the seqno as an indicator for that and a rolling counter
738 * for the remaining 15 bits.
739 */
740 #define CT_SEQNO_MASK GENMASK(14, 0)
741 #define CT_SEQNO_UNTRACKED BIT(15)
next_ct_seqno(struct xe_guc_ct * ct,bool is_g2h_fence)742 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)
743 {
744 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;
745
746 if (!is_g2h_fence)
747 seqno |= CT_SEQNO_UNTRACKED;
748
749 return seqno;
750 }
751
__guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)752 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
753 u32 len, u32 g2h_len, u32 num_g2h,
754 struct g2h_fence *g2h_fence)
755 {
756 struct xe_gt *gt __maybe_unused = ct_to_gt(ct);
757 u16 seqno;
758 int ret;
759
760 xe_gt_assert(gt, xe_guc_ct_initialized(ct));
761 xe_gt_assert(gt, !g2h_len || !g2h_fence);
762 xe_gt_assert(gt, !num_g2h || !g2h_fence);
763 xe_gt_assert(gt, !g2h_len || num_g2h);
764 xe_gt_assert(gt, g2h_len || !num_g2h);
765 lockdep_assert_held(&ct->lock);
766
767 if (unlikely(ct->ctbs.h2g.info.broken)) {
768 ret = -EPIPE;
769 goto out;
770 }
771
772 if (ct->state == XE_GUC_CT_STATE_DISABLED) {
773 ret = -ENODEV;
774 goto out;
775 }
776
777 if (ct->state == XE_GUC_CT_STATE_STOPPED) {
778 ret = -ECANCELED;
779 goto out;
780 }
781
782 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
783
784 if (g2h_fence) {
785 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
786 num_g2h = 1;
787
788 if (g2h_fence_needs_alloc(g2h_fence)) {
789 g2h_fence->seqno = next_ct_seqno(ct, true);
790 ret = xa_err(xa_store(&ct->fence_lookup,
791 g2h_fence->seqno, g2h_fence,
792 GFP_ATOMIC));
793 if (ret)
794 goto out;
795 }
796
797 seqno = g2h_fence->seqno;
798 } else {
799 seqno = next_ct_seqno(ct, false);
800 }
801
802 if (g2h_len)
803 spin_lock_irq(&ct->fast_lock);
804 retry:
805 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
806 if (unlikely(ret))
807 goto out_unlock;
808
809 ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
810 if (unlikely(ret)) {
811 if (ret == -EAGAIN)
812 goto retry;
813 goto out_unlock;
814 }
815
816 __g2h_reserve_space(ct, g2h_len, num_g2h);
817 xe_guc_notify(ct_to_guc(ct));
818 out_unlock:
819 if (g2h_len)
820 spin_unlock_irq(&ct->fast_lock);
821 out:
822 return ret;
823 }
824
kick_reset(struct xe_guc_ct * ct)825 static void kick_reset(struct xe_guc_ct *ct)
826 {
827 xe_gt_reset_async(ct_to_gt(ct));
828 }
829
830 static int dequeue_one_g2h(struct xe_guc_ct *ct);
831
guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)832 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
833 u32 g2h_len, u32 num_g2h,
834 struct g2h_fence *g2h_fence)
835 {
836 struct xe_device *xe = ct_to_xe(ct);
837 struct xe_gt *gt = ct_to_gt(ct);
838 unsigned int sleep_period_ms = 1;
839 int ret;
840
841 xe_gt_assert(gt, !g2h_len || !g2h_fence);
842 lockdep_assert_held(&ct->lock);
843 xe_device_assert_mem_access(ct_to_xe(ct));
844
845 try_again:
846 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
847 g2h_fence);
848
849 /*
850 * We wait to try to restore credits for about 1 second before bailing.
851 * In the case of H2G credits we have no choice but just to wait for the
852 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
853 * the case of G2H we process any G2H in the channel, hopefully freeing
854 * credits as we consume the G2H messages.
855 */
856 if (unlikely(ret == -EBUSY &&
857 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {
858 struct guc_ctb *h2g = &ct->ctbs.h2g;
859
860 if (sleep_period_ms == 1024)
861 goto broken;
862
863 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
864 h2g->info.size,
865 h2g->info.space,
866 len + GUC_CTB_HDR_LEN);
867 msleep(sleep_period_ms);
868 sleep_period_ms <<= 1;
869
870 goto try_again;
871 } else if (unlikely(ret == -EBUSY)) {
872 struct xe_device *xe = ct_to_xe(ct);
873 struct guc_ctb *g2h = &ct->ctbs.g2h;
874
875 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,
876 desc_read(xe, g2h, tail),
877 g2h->info.size,
878 g2h->info.space,
879 g2h_fence ?
880 GUC_CTB_HXG_MSG_MAX_LEN :
881 g2h_len);
882
883 #define g2h_avail(ct) \
884 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
885 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
886 g2h_avail(ct), HZ))
887 goto broken;
888 #undef g2h_avail
889
890 ret = dequeue_one_g2h(ct);
891 if (ret < 0) {
892 if (ret != -ECANCELED)
893 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)",
894 ERR_PTR(ret));
895 goto broken;
896 }
897
898 goto try_again;
899 }
900
901 return ret;
902
903 broken:
904 xe_gt_err(gt, "No forward process on H2G, reset required\n");
905 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK);
906
907 return -EDEADLK;
908 }
909
guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)910 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
911 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
912 {
913 int ret;
914
915 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
916
917 mutex_lock(&ct->lock);
918 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
919 mutex_unlock(&ct->lock);
920
921 return ret;
922 }
923
xe_guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)924 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
925 u32 g2h_len, u32 num_g2h)
926 {
927 int ret;
928
929 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
930 if (ret == -EDEADLK)
931 kick_reset(ct);
932
933 return ret;
934 }
935
xe_guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)936 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
937 u32 g2h_len, u32 num_g2h)
938 {
939 int ret;
940
941 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
942 if (ret == -EDEADLK)
943 kick_reset(ct);
944
945 return ret;
946 }
947
xe_guc_ct_send_g2h_handler(struct xe_guc_ct * ct,const u32 * action,u32 len)948 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
949 {
950 int ret;
951
952 lockdep_assert_held(&ct->lock);
953
954 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
955 if (ret == -EDEADLK)
956 kick_reset(ct);
957
958 return ret;
959 }
960
961 /*
962 * Check if a GT reset is in progress or will occur and if GT reset brought the
963 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
964 */
retry_failure(struct xe_guc_ct * ct,int ret)965 static bool retry_failure(struct xe_guc_ct *ct, int ret)
966 {
967 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
968 return false;
969
970 #define ct_alive(ct) \
971 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \
972 !ct->ctbs.g2h.info.broken)
973 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
974 return false;
975 #undef ct_alive
976
977 return true;
978 }
979
guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer,bool no_fail)980 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
981 u32 *response_buffer, bool no_fail)
982 {
983 struct xe_gt *gt = ct_to_gt(ct);
984 struct g2h_fence g2h_fence;
985 int ret = 0;
986
987 /*
988 * We use a fence to implement blocking sends / receiving response data.
989 * The seqno of the fence is sent in the H2G, returned in the G2H, and
990 * an xarray is used as storage media with the seqno being to key.
991 * Fields in the fence hold success, failure, retry status and the
992 * response data. Safe to allocate on the stack as the xarray is the
993 * only reference and it cannot be present after this function exits.
994 */
995 retry:
996 g2h_fence_init(&g2h_fence, response_buffer);
997 retry_same_fence:
998 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
999 if (unlikely(ret == -ENOMEM)) {
1000 /* Retry allocation /w GFP_KERNEL */
1001 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno,
1002 &g2h_fence, GFP_KERNEL));
1003 if (ret)
1004 return ret;
1005
1006 goto retry_same_fence;
1007 } else if (unlikely(ret)) {
1008 if (ret == -EDEADLK)
1009 kick_reset(ct);
1010
1011 if (no_fail && retry_failure(ct, ret))
1012 goto retry_same_fence;
1013
1014 if (!g2h_fence_needs_alloc(&g2h_fence))
1015 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
1016
1017 return ret;
1018 }
1019
1020 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
1021
1022 if (!ret) {
1023 LNL_FLUSH_WORK(&ct->g2h_worker);
1024 if (g2h_fence.done) {
1025 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",
1026 g2h_fence.seqno, action[0]);
1027 ret = 1;
1028 }
1029 }
1030
1031 /*
1032 * Occasionally it is seen that the G2H worker starts running after a delay of more than
1033 * a second even after being queued and activated by the Linux workqueue subsystem. This
1034 * leads to G2H timeout error. The root cause of issue lies with scheduling latency of
1035 * Lunarlake Hybrid CPU. Issue dissappears if we disable Lunarlake atom cores from BIOS
1036 * and this is beyond xe kmd.
1037 *
1038 * TODO: Drop this change once workqueue scheduling delay issue is fixed on LNL Hybrid CPU.
1039 */
1040 if (!ret) {
1041 flush_work(&ct->g2h_worker);
1042 if (g2h_fence.done) {
1043 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",
1044 g2h_fence.seqno, action[0]);
1045 ret = 1;
1046 }
1047 }
1048
1049 /*
1050 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on
1051 * the stack, since we have no clue if it will fire after the timeout before we can erase
1052 * from the xa. Also we have some dependent loads and stores below for which we need the
1053 * correct ordering, and we lack the needed barriers.
1054 */
1055 mutex_lock(&ct->lock);
1056 if (!ret) {
1057 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",
1058 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));
1059 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
1060 mutex_unlock(&ct->lock);
1061 return -ETIME;
1062 }
1063
1064 if (g2h_fence.retry) {
1065 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
1066 action[0], g2h_fence.reason);
1067 mutex_unlock(&ct->lock);
1068 goto retry;
1069 }
1070 if (g2h_fence.fail) {
1071 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
1072 action[0], g2h_fence.error, g2h_fence.hint);
1073 ret = -EIO;
1074 }
1075
1076 if (ret > 0)
1077 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data;
1078
1079 mutex_unlock(&ct->lock);
1080
1081 return ret;
1082 }
1083
1084 /**
1085 * xe_guc_ct_send_recv - Send and receive HXG to the GuC
1086 * @ct: the &xe_guc_ct
1087 * @action: the dword array with `HXG Request`_ message (can't be NULL)
1088 * @len: length of the `HXG Request`_ message (in dwords, can't be 0)
1089 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)
1090 *
1091 * Send a `HXG Request`_ message to the GuC over CT communication channel and
1092 * blocks until GuC replies with a `HXG Response`_ message.
1093 *
1094 * For non-blocking communication with GuC use xe_guc_ct_send().
1095 *
1096 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.
1097 *
1098 * Return: response length (in dwords) if &response_buffer was not NULL, or
1099 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or
1100 * a negative error code on failure.
1101 */
xe_guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1102 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
1103 u32 *response_buffer)
1104 {
1105 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);
1106 return guc_ct_send_recv(ct, action, len, response_buffer, false);
1107 }
1108
xe_guc_ct_send_recv_no_fail(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1109 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
1110 u32 len, u32 *response_buffer)
1111 {
1112 return guc_ct_send_recv(ct, action, len, response_buffer, true);
1113 }
1114
msg_to_hxg(u32 * msg)1115 static u32 *msg_to_hxg(u32 *msg)
1116 {
1117 return msg + GUC_CTB_MSG_MIN_LEN;
1118 }
1119
msg_len_to_hxg_len(u32 len)1120 static u32 msg_len_to_hxg_len(u32 len)
1121 {
1122 return len - GUC_CTB_MSG_MIN_LEN;
1123 }
1124
parse_g2h_event(struct xe_guc_ct * ct,u32 * msg,u32 len)1125 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
1126 {
1127 u32 *hxg = msg_to_hxg(msg);
1128 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1129
1130 lockdep_assert_held(&ct->lock);
1131
1132 switch (action) {
1133 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1134 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1135 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1136 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1137 g2h_release_space(ct, len);
1138 }
1139
1140 return 0;
1141 }
1142
parse_g2h_response(struct xe_guc_ct * ct,u32 * msg,u32 len)1143 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
1144 {
1145 struct xe_gt *gt = ct_to_gt(ct);
1146 u32 *hxg = msg_to_hxg(msg);
1147 u32 hxg_len = msg_len_to_hxg_len(len);
1148 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
1149 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1150 struct g2h_fence *g2h_fence;
1151
1152 lockdep_assert_held(&ct->lock);
1153
1154 /*
1155 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.
1156 * Those messages should never fail, so if we do get an error back it
1157 * means we're likely doing an illegal operation and the GuC is
1158 * rejecting it. We have no way to inform the code that submitted the
1159 * H2G that the message was rejected, so we need to escalate the
1160 * failure to trigger a reset.
1161 */
1162 if (fence & CT_SEQNO_UNTRACKED) {
1163 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)
1164 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",
1165 fence,
1166 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),
1167 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));
1168 else
1169 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",
1170 type, fence);
1171 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE);
1172
1173 return -EPROTO;
1174 }
1175
1176 g2h_fence = xa_erase(&ct->fence_lookup, fence);
1177 if (unlikely(!g2h_fence)) {
1178 /* Don't tear down channel, as send could've timed out */
1179 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */
1180 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);
1181 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1182 return 0;
1183 }
1184
1185 xe_gt_assert(gt, fence == g2h_fence->seqno);
1186
1187 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
1188 g2h_fence->fail = true;
1189 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
1190 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);
1191 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1192 g2h_fence->retry = true;
1193 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
1194 } else if (g2h_fence->response_buffer) {
1195 g2h_fence->response_len = hxg_len;
1196 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
1197 } else {
1198 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
1199 }
1200
1201 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1202
1203 g2h_fence->done = true;
1204 smp_mb();
1205
1206 wake_up_all(&ct->g2h_fence_wq);
1207
1208 return 0;
1209 }
1210
parse_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1211 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1212 {
1213 struct xe_gt *gt = ct_to_gt(ct);
1214 u32 *hxg = msg_to_hxg(msg);
1215 u32 origin, type;
1216 int ret;
1217
1218 lockdep_assert_held(&ct->lock);
1219
1220 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1221 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1222 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
1223 origin);
1224 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN);
1225
1226 return -EPROTO;
1227 }
1228
1229 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1230 switch (type) {
1231 case GUC_HXG_TYPE_EVENT:
1232 ret = parse_g2h_event(ct, msg, len);
1233 break;
1234 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1235 case GUC_HXG_TYPE_RESPONSE_FAILURE:
1236 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1237 ret = parse_g2h_response(ct, msg, len);
1238 break;
1239 default:
1240 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
1241 type);
1242 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE);
1243
1244 ret = -EOPNOTSUPP;
1245 }
1246
1247 return ret;
1248 }
1249
process_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1250 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1251 {
1252 struct xe_guc *guc = ct_to_guc(ct);
1253 struct xe_gt *gt = ct_to_gt(ct);
1254 u32 hxg_len = msg_len_to_hxg_len(len);
1255 u32 *hxg = msg_to_hxg(msg);
1256 u32 action, adj_len;
1257 u32 *payload;
1258 int ret = 0;
1259
1260 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1261 return 0;
1262
1263 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1264 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;
1265 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;
1266
1267 switch (action) {
1268 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1269 ret = xe_guc_sched_done_handler(guc, payload, adj_len);
1270 break;
1271 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1272 ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
1273 break;
1274 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1275 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
1276 break;
1277 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1278 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
1279 adj_len);
1280 break;
1281 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1282 /* Selftest only at the moment */
1283 break;
1284 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1285 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1286 /* FIXME: Handle this */
1287 break;
1288 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
1289 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
1290 adj_len);
1291 break;
1292 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1293 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1294 break;
1295 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1296 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1297 adj_len);
1298 break;
1299 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:
1300 ret = xe_guc_access_counter_notify_handler(guc, payload,
1301 adj_len);
1302 break;
1303 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
1304 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
1305 break;
1306 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
1307 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
1308 break;
1309 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
1310 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
1311 break;
1312 case GUC_ACTION_GUC2PF_ADVERSE_EVENT:
1313 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);
1314 break;
1315 default:
1316 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1317 }
1318
1319 if (ret) {
1320 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1321 action, ERR_PTR(ret));
1322 CT_DEAD(ct, NULL, PROCESS_FAILED);
1323 }
1324
1325 return 0;
1326 }
1327
g2h_read(struct xe_guc_ct * ct,u32 * msg,bool fast_path)1328 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
1329 {
1330 struct xe_device *xe = ct_to_xe(ct);
1331 struct xe_gt *gt = ct_to_gt(ct);
1332 struct guc_ctb *g2h = &ct->ctbs.g2h;
1333 u32 tail, head, len, desc_status;
1334 s32 avail;
1335 u32 action;
1336 u32 *hxg;
1337
1338 xe_gt_assert(gt, xe_guc_ct_initialized(ct));
1339 lockdep_assert_held(&ct->fast_lock);
1340
1341 if (ct->state == XE_GUC_CT_STATE_DISABLED)
1342 return -ENODEV;
1343
1344 if (ct->state == XE_GUC_CT_STATE_STOPPED)
1345 return -ECANCELED;
1346
1347 if (g2h->info.broken)
1348 return -EPIPE;
1349
1350 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
1351
1352 desc_status = desc_read(xe, g2h, status);
1353 if (desc_status) {
1354 if (desc_status & GUC_CTB_STATUS_DISABLED) {
1355 /*
1356 * Potentially valid if a CLIENT_RESET request resulted in
1357 * contexts/engines being reset. But should never happen as
1358 * no contexts should be active when CLIENT_RESET is sent.
1359 */
1360 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n");
1361 desc_status &= ~GUC_CTB_STATUS_DISABLED;
1362 }
1363
1364 if (desc_status) {
1365 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status);
1366 goto corrupted;
1367 }
1368 }
1369
1370 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
1371 u32 desc_tail = desc_read(xe, g2h, tail);
1372 /*
1373 u32 desc_head = desc_read(xe, g2h, head);
1374
1375 * info.head and desc_head are updated back-to-back at the end of
1376 * this function and nowhere else. Hence, they cannot be different
1377 * unless two g2h_read calls are running concurrently. Which is not
1378 * possible because it is guarded by ct->fast_lock. And yet, some
1379 * discrete platforms are reguarly hitting this error :(.
1380 *
1381 * desc_head rolling backwards shouldn't cause any noticeable
1382 * problems - just a delay in GuC being allowed to proceed past that
1383 * point in the queue. So for now, just disable the error until it
1384 * can be root caused.
1385 *
1386 if (g2h->info.head != desc_head) {
1387 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH);
1388 xe_gt_err(gt, "CT read: head was modified %u != %u\n",
1389 desc_head, g2h->info.head);
1390 goto corrupted;
1391 }
1392 */
1393
1394 if (g2h->info.head > g2h->info.size) {
1395 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1396 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n",
1397 g2h->info.head, g2h->info.size);
1398 goto corrupted;
1399 }
1400
1401 if (desc_tail >= g2h->info.size) {
1402 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1403 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n",
1404 desc_tail, g2h->info.size);
1405 goto corrupted;
1406 }
1407 }
1408
1409 /* Calculate DW available to read */
1410 tail = desc_read(xe, g2h, tail);
1411 avail = tail - g2h->info.head;
1412 if (unlikely(avail == 0))
1413 return 0;
1414
1415 if (avail < 0)
1416 avail += g2h->info.size;
1417
1418 /* Read header */
1419 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
1420 sizeof(u32));
1421 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
1422 if (len > avail) {
1423 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
1424 avail, len);
1425 goto corrupted;
1426 }
1427
1428 head = (g2h->info.head + 1) % g2h->info.size;
1429 avail = len - 1;
1430
1431 /* Read G2H message */
1432 if (avail + head > g2h->info.size) {
1433 u32 avail_til_wrap = g2h->info.size - head;
1434
1435 xe_map_memcpy_from(xe, msg + 1,
1436 &g2h->cmds, sizeof(u32) * head,
1437 avail_til_wrap * sizeof(u32));
1438 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
1439 &g2h->cmds, 0,
1440 (avail - avail_til_wrap) * sizeof(u32));
1441 } else {
1442 xe_map_memcpy_from(xe, msg + 1,
1443 &g2h->cmds, sizeof(u32) * head,
1444 avail * sizeof(u32));
1445 }
1446
1447 hxg = msg_to_hxg(msg);
1448 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1449
1450 if (fast_path) {
1451 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1452 return 0;
1453
1454 switch (action) {
1455 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1456 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1457 break; /* Process these in fast-path */
1458 default:
1459 return 0;
1460 }
1461 }
1462
1463 /* Update local / descriptor header */
1464 g2h->info.head = (head + avail) % g2h->info.size;
1465 desc_write(xe, g2h, head, g2h->info.head);
1466
1467 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,
1468 action, len, g2h->info.head, tail);
1469
1470 return len;
1471
1472 corrupted:
1473 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ);
1474 return -EPROTO;
1475 }
1476
g2h_fast_path(struct xe_guc_ct * ct,u32 * msg,u32 len)1477 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1478 {
1479 struct xe_gt *gt = ct_to_gt(ct);
1480 struct xe_guc *guc = ct_to_guc(ct);
1481 u32 hxg_len = msg_len_to_hxg_len(len);
1482 u32 *hxg = msg_to_hxg(msg);
1483 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1484 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;
1485 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1486 int ret = 0;
1487
1488 switch (action) {
1489 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1490 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1491 break;
1492 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1493 __g2h_release_space(ct, len);
1494 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1495 adj_len);
1496 break;
1497 default:
1498 xe_gt_warn(gt, "NOT_POSSIBLE");
1499 }
1500
1501 if (ret) {
1502 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1503 action, ERR_PTR(ret));
1504 CT_DEAD(ct, NULL, FAST_G2H);
1505 }
1506 }
1507
1508 /**
1509 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1510 * @ct: GuC CT object
1511 *
1512 * Anything related to page faults is critical for performance, process these
1513 * critical G2H in the IRQ. This is safe as these handlers either just wake up
1514 * waiters or queue another worker.
1515 */
xe_guc_ct_fast_path(struct xe_guc_ct * ct)1516 void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1517 {
1518 struct xe_device *xe = ct_to_xe(ct);
1519 bool ongoing;
1520 int len;
1521
1522 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1523 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1524 return;
1525
1526 spin_lock(&ct->fast_lock);
1527 do {
1528 len = g2h_read(ct, ct->fast_msg, true);
1529 if (len > 0)
1530 g2h_fast_path(ct, ct->fast_msg, len);
1531 } while (len > 0);
1532 spin_unlock(&ct->fast_lock);
1533
1534 if (ongoing)
1535 xe_pm_runtime_put(xe);
1536 }
1537
1538 /* Returns less than zero on error, 0 on done, 1 on more available */
dequeue_one_g2h(struct xe_guc_ct * ct)1539 static int dequeue_one_g2h(struct xe_guc_ct *ct)
1540 {
1541 int len;
1542 int ret;
1543
1544 lockdep_assert_held(&ct->lock);
1545
1546 spin_lock_irq(&ct->fast_lock);
1547 len = g2h_read(ct, ct->msg, false);
1548 spin_unlock_irq(&ct->fast_lock);
1549 if (len <= 0)
1550 return len;
1551
1552 ret = parse_g2h_msg(ct, ct->msg, len);
1553 if (unlikely(ret < 0))
1554 return ret;
1555
1556 ret = process_g2h_msg(ct, ct->msg, len);
1557 if (unlikely(ret < 0))
1558 return ret;
1559
1560 return 1;
1561 }
1562
receive_g2h(struct xe_guc_ct * ct)1563 static void receive_g2h(struct xe_guc_ct *ct)
1564 {
1565 bool ongoing;
1566 int ret;
1567
1568 /*
1569 * Normal users must always hold mem_access.ref around CT calls. However
1570 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1571 * at this stage we can't rely on mem_access.ref and even the
1572 * callback_task will be different than current. For such cases we just
1573 * need to ensure we always process the responses from any blocking
1574 * ct_send requests or where we otherwise expect some response when
1575 * initiated from those callbacks (which will need to wait for the below
1576 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if
1577 * the device has suspended to the point that the CT communication has
1578 * been disabled.
1579 *
1580 * If we are inside the runtime pm callback, we can be the only task
1581 * still issuing CT requests (since that requires having the
1582 * mem_access.ref). It seems like it might in theory be possible to
1583 * receive unsolicited events from the GuC just as we are
1584 * suspending-resuming, but those will currently anyway be lost when
1585 * eventually exiting from suspend, hence no need to wake up the device
1586 * here. If we ever need something stronger than get_if_ongoing() then
1587 * we need to be careful with blocking the pm callbacks from getting CT
1588 * responses, if the worker here is blocked on those callbacks
1589 * completing, creating a deadlock.
1590 */
1591 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1592 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1593 return;
1594
1595 do {
1596 mutex_lock(&ct->lock);
1597 ret = dequeue_one_g2h(ct);
1598 mutex_unlock(&ct->lock);
1599
1600 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1601 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret);
1602 CT_DEAD(ct, NULL, G2H_RECV);
1603 kick_reset(ct);
1604 }
1605 } while (ret == 1);
1606
1607 if (ongoing)
1608 xe_pm_runtime_put(ct_to_xe(ct));
1609 }
1610
g2h_worker_func(struct work_struct * w)1611 static void g2h_worker_func(struct work_struct *w)
1612 {
1613 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1614
1615 receive_g2h(ct);
1616 }
1617
guc_ctb_snapshot_capture(struct xe_device * xe,struct guc_ctb * ctb,struct guc_ctb_snapshot * snapshot,bool atomic)1618 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1619 struct guc_ctb_snapshot *snapshot,
1620 bool atomic)
1621 {
1622 u32 head, tail;
1623
1624 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1625 sizeof(struct guc_ct_buffer_desc));
1626 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1627
1628 snapshot->cmds = kmalloc_array(ctb->info.size, sizeof(u32),
1629 atomic ? GFP_ATOMIC : GFP_KERNEL);
1630 if (!snapshot->cmds) {
1631 drm_err(&xe->drm, "Skipping CTB commands snapshot. Only CT info will be available.\n");
1632 return;
1633 }
1634
1635 head = snapshot->desc.head;
1636 tail = snapshot->desc.tail;
1637
1638 if (head != tail) {
1639 struct iosys_map map =
1640 IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32));
1641
1642 while (head != tail) {
1643 snapshot->cmds[head] = xe_map_rd(xe, &map, 0, u32);
1644 ++head;
1645 if (head == ctb->info.size) {
1646 head = 0;
1647 map = ctb->cmds;
1648 } else {
1649 iosys_map_incr(&map, sizeof(u32));
1650 }
1651 }
1652 }
1653 }
1654
guc_ctb_snapshot_print(struct guc_ctb_snapshot * snapshot,struct drm_printer * p)1655 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1656 struct drm_printer *p)
1657 {
1658 u32 head, tail;
1659
1660 drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1661 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1662 drm_printf(p, "\thead: %d\n", snapshot->info.head);
1663 drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1664 drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1665 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1666 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1667 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1668 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1669
1670 if (!snapshot->cmds)
1671 return;
1672
1673 head = snapshot->desc.head;
1674 tail = snapshot->desc.tail;
1675
1676 while (head != tail) {
1677 drm_printf(p, "\tcmd[%d]: 0x%08x\n", head,
1678 snapshot->cmds[head]);
1679 ++head;
1680 if (head == snapshot->info.size)
1681 head = 0;
1682 }
1683 }
1684
guc_ctb_snapshot_free(struct guc_ctb_snapshot * snapshot)1685 static void guc_ctb_snapshot_free(struct guc_ctb_snapshot *snapshot)
1686 {
1687 kfree(snapshot->cmds);
1688 }
1689
1690 /**
1691 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1692 * @ct: GuC CT object.
1693 * @atomic: Boolean to indicate if this is called from atomic context like
1694 * reset or CTB handler or from some regular path like debugfs.
1695 *
1696 * This can be printed out in a later stage like during dev_coredump
1697 * analysis.
1698 *
1699 * Returns: a GuC CT snapshot object that must be freed by the caller
1700 * by using `xe_guc_ct_snapshot_free`.
1701 */
xe_guc_ct_snapshot_capture(struct xe_guc_ct * ct,bool atomic)1702 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,
1703 bool atomic)
1704 {
1705 struct xe_device *xe = ct_to_xe(ct);
1706 struct xe_guc_ct_snapshot *snapshot;
1707
1708 snapshot = kzalloc(sizeof(*snapshot),
1709 atomic ? GFP_ATOMIC : GFP_KERNEL);
1710
1711 if (!snapshot) {
1712 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n");
1713 return NULL;
1714 }
1715
1716 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
1717 snapshot->ct_enabled = true;
1718 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1719 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,
1720 &snapshot->h2g, atomic);
1721 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h,
1722 &snapshot->g2h, atomic);
1723 }
1724
1725 return snapshot;
1726 }
1727
1728 /**
1729 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1730 * @snapshot: GuC CT snapshot object.
1731 * @p: drm_printer where it will be printed out.
1732 *
1733 * This function prints out a given GuC CT snapshot object.
1734 */
xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot * snapshot,struct drm_printer * p)1735 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1736 struct drm_printer *p)
1737 {
1738 if (!snapshot)
1739 return;
1740
1741 if (snapshot->ct_enabled) {
1742 drm_puts(p, "H2G CTB (all sizes in DW):\n");
1743 guc_ctb_snapshot_print(&snapshot->h2g, p);
1744
1745 drm_puts(p, "\nG2H CTB (all sizes in DW):\n");
1746 guc_ctb_snapshot_print(&snapshot->g2h, p);
1747
1748 drm_printf(p, "\tg2h outstanding: %d\n",
1749 snapshot->g2h_outstanding);
1750 } else {
1751 drm_puts(p, "CT disabled\n");
1752 }
1753 }
1754
1755 /**
1756 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1757 * @snapshot: GuC CT snapshot object.
1758 *
1759 * This function free all the memory that needed to be allocated at capture
1760 * time.
1761 */
xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot * snapshot)1762 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
1763 {
1764 if (!snapshot)
1765 return;
1766
1767 guc_ctb_snapshot_free(&snapshot->h2g);
1768 guc_ctb_snapshot_free(&snapshot->g2h);
1769 kfree(snapshot);
1770 }
1771
1772 /**
1773 * xe_guc_ct_print - GuC CT Print.
1774 * @ct: GuC CT.
1775 * @p: drm_printer where it will be printed out.
1776 *
1777 * This function quickly capture a snapshot and immediately print it out.
1778 */
xe_guc_ct_print(struct xe_guc_ct * ct,struct drm_printer * p)1779 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p)
1780 {
1781 struct xe_guc_ct_snapshot *snapshot;
1782
1783 snapshot = xe_guc_ct_snapshot_capture(ct, false);
1784 xe_guc_ct_snapshot_print(snapshot, p);
1785 xe_guc_ct_snapshot_free(snapshot);
1786 }
1787
1788 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
ct_dead_capture(struct xe_guc_ct * ct,struct guc_ctb * ctb,u32 reason_code)1789 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code)
1790 {
1791 struct xe_guc_log_snapshot *snapshot_log;
1792 struct xe_guc_ct_snapshot *snapshot_ct;
1793 struct xe_guc *guc = ct_to_guc(ct);
1794 unsigned long flags;
1795 bool have_capture;
1796
1797 if (ctb)
1798 ctb->info.broken = true;
1799
1800 /* Ignore further errors after the first dump until a reset */
1801 if (ct->dead.reported)
1802 return;
1803
1804 spin_lock_irqsave(&ct->dead.lock, flags);
1805
1806 /* And only capture one dump at a time */
1807 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE);
1808 ct->dead.reason |= (1 << reason_code) |
1809 (1 << CT_DEAD_STATE_CAPTURE);
1810
1811 spin_unlock_irqrestore(&ct->dead.lock, flags);
1812
1813 if (have_capture)
1814 return;
1815
1816 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true);
1817 snapshot_ct = xe_guc_ct_snapshot_capture((ct), true);
1818
1819 spin_lock_irqsave(&ct->dead.lock, flags);
1820
1821 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) {
1822 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n");
1823 xe_guc_log_snapshot_free(snapshot_log);
1824 xe_guc_ct_snapshot_free(snapshot_ct);
1825 } else {
1826 ct->dead.snapshot_log = snapshot_log;
1827 ct->dead.snapshot_ct = snapshot_ct;
1828 }
1829
1830 spin_unlock_irqrestore(&ct->dead.lock, flags);
1831
1832 queue_work(system_unbound_wq, &(ct)->dead.worker);
1833 }
1834
ct_dead_print(struct xe_dead_ct * dead)1835 static void ct_dead_print(struct xe_dead_ct *dead)
1836 {
1837 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead);
1838 struct xe_device *xe = ct_to_xe(ct);
1839 struct xe_gt *gt = ct_to_gt(ct);
1840 static int g_count;
1841 struct drm_printer ip = xe_gt_info_printer(gt);
1842 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count);
1843
1844 if (!dead->reason) {
1845 xe_gt_err(gt, "CTB is dead for no reason!?\n");
1846 return;
1847 }
1848
1849 drm_printf(&lp, "CTB is dead - reason=0x%X\n", dead->reason);
1850
1851 /* Can't generate a genuine core dump at this point, so just do the good bits */
1852 drm_puts(&lp, "**** Xe Device Coredump ****\n");
1853 xe_device_snapshot_print(xe, &lp);
1854
1855 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id);
1856 drm_printf(&lp, "\tTile: %d\n", gt->tile->id);
1857
1858 drm_puts(&lp, "**** GuC Log ****\n");
1859 xe_guc_log_snapshot_print(dead->snapshot_log, &lp);
1860
1861 drm_puts(&lp, "**** GuC CT ****\n");
1862 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp);
1863
1864 drm_puts(&lp, "Done.\n");
1865 }
1866
ct_dead_worker_func(struct work_struct * w)1867 static void ct_dead_worker_func(struct work_struct *w)
1868 {
1869 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker);
1870
1871 if (!ct->dead.reported) {
1872 ct->dead.reported = true;
1873 ct_dead_print(&ct->dead);
1874 }
1875
1876 spin_lock_irq(&ct->dead.lock);
1877
1878 xe_guc_log_snapshot_free(ct->dead.snapshot_log);
1879 ct->dead.snapshot_log = NULL;
1880 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct);
1881 ct->dead.snapshot_ct = NULL;
1882
1883 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) {
1884 /* A reset has occurred so re-arm the error reporting */
1885 ct->dead.reason = 0;
1886 ct->dead.reported = false;
1887 }
1888
1889 spin_unlock_irq(&ct->dead.lock);
1890 }
1891 #endif
1892