1 /*
2 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 * Authors:
24 * Zhiyuan Lv <zhiyuan.lv@intel.com>
25 * Zhi Wang <zhi.a.wang@intel.com>
26 *
27 * Contributors:
28 * Min He <min.he@intel.com>
29 * Bing Niu <bing.niu@intel.com>
30 * Ping Gao <ping.a.gao@intel.com>
31 * Tina Zhang <tina.zhang@intel.com>
32 *
33 */
34
35 #include "i915_drv.h"
36 #include "gvt.h"
37
38 #define _EL_OFFSET_STATUS 0x234
39 #define _EL_OFFSET_STATUS_BUF 0x370
40 #define _EL_OFFSET_STATUS_PTR 0x3A0
41
42 #define execlist_ring_mmio(e, offset) ((e)->mmio_base + (offset))
43
44 #define valid_context(ctx) ((ctx)->valid)
45 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
46 ((a)->lrca == (b)->lrca))
47
48 static int context_switch_events[] = {
49 [RCS0] = RCS_AS_CONTEXT_SWITCH,
50 [BCS0] = BCS_AS_CONTEXT_SWITCH,
51 [VCS0] = VCS_AS_CONTEXT_SWITCH,
52 [VCS1] = VCS2_AS_CONTEXT_SWITCH,
53 [VECS0] = VECS_AS_CONTEXT_SWITCH,
54 };
55
to_context_switch_event(const struct intel_engine_cs * engine)56 static int to_context_switch_event(const struct intel_engine_cs *engine)
57 {
58 if (WARN_ON(engine->id >= ARRAY_SIZE(context_switch_events)))
59 return -EINVAL;
60
61 return context_switch_events[engine->id];
62 }
63
switch_virtual_execlist_slot(struct intel_vgpu_execlist * execlist)64 static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist)
65 {
66 gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n",
67 execlist->running_slot ?
68 execlist->running_slot->index : -1,
69 execlist->running_context ?
70 execlist->running_context->context_id : 0,
71 execlist->pending_slot ?
72 execlist->pending_slot->index : -1);
73
74 execlist->running_slot = execlist->pending_slot;
75 execlist->pending_slot = NULL;
76 execlist->running_context = execlist->running_context ?
77 &execlist->running_slot->ctx[0] : NULL;
78
79 gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n",
80 execlist->running_slot ?
81 execlist->running_slot->index : -1,
82 execlist->running_context ?
83 execlist->running_context->context_id : 0,
84 execlist->pending_slot ?
85 execlist->pending_slot->index : -1);
86 }
87
emulate_execlist_status(struct intel_vgpu_execlist * execlist)88 static void emulate_execlist_status(struct intel_vgpu_execlist *execlist)
89 {
90 struct intel_vgpu_execlist_slot *running = execlist->running_slot;
91 struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
92 struct execlist_ctx_descriptor_format *desc = execlist->running_context;
93 struct intel_vgpu *vgpu = execlist->vgpu;
94 struct execlist_status_format status;
95 u32 status_reg =
96 execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
97
98 status.ldw = vgpu_vreg(vgpu, status_reg);
99 status.udw = vgpu_vreg(vgpu, status_reg + 4);
100
101 if (running) {
102 status.current_execlist_pointer = !!running->index;
103 status.execlist_write_pointer = !!!running->index;
104 status.execlist_0_active = status.execlist_0_valid =
105 !!!(running->index);
106 status.execlist_1_active = status.execlist_1_valid =
107 !!(running->index);
108 } else {
109 status.context_id = 0;
110 status.execlist_0_active = status.execlist_0_valid = 0;
111 status.execlist_1_active = status.execlist_1_valid = 0;
112 }
113
114 status.context_id = desc ? desc->context_id : 0;
115 status.execlist_queue_full = !!(pending);
116
117 vgpu_vreg(vgpu, status_reg) = status.ldw;
118 vgpu_vreg(vgpu, status_reg + 4) = status.udw;
119
120 gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n",
121 vgpu->id, status_reg, status.ldw, status.udw);
122 }
123
emulate_csb_update(struct intel_vgpu_execlist * execlist,struct execlist_context_status_format * status,bool trigger_interrupt_later)124 static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
125 struct execlist_context_status_format *status,
126 bool trigger_interrupt_later)
127 {
128 struct intel_vgpu *vgpu = execlist->vgpu;
129 struct execlist_context_status_pointer_format ctx_status_ptr;
130 u32 write_pointer;
131 u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
132 unsigned long hwsp_gpa;
133
134 ctx_status_ptr_reg =
135 execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_PTR);
136 ctx_status_buf_reg =
137 execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_BUF);
138
139 ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
140
141 write_pointer = ctx_status_ptr.write_ptr;
142
143 if (write_pointer == 0x7)
144 write_pointer = 0;
145 else {
146 ++write_pointer;
147 write_pointer %= 0x6;
148 }
149
150 offset = ctx_status_buf_reg + write_pointer * 8;
151
152 vgpu_vreg(vgpu, offset) = status->ldw;
153 vgpu_vreg(vgpu, offset + 4) = status->udw;
154
155 ctx_status_ptr.write_ptr = write_pointer;
156 vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
157
158 /* Update the CSB and CSB write pointer in HWSP */
159 hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
160 vgpu->hws_pga[execlist->engine->id]);
161 if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) {
162 intel_gvt_hypervisor_write_gpa(vgpu,
163 hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + write_pointer * 8,
164 status, 8);
165 intel_gvt_hypervisor_write_gpa(vgpu,
166 hwsp_gpa + intel_hws_csb_write_index(execlist->engine->i915) * 4,
167 &write_pointer, 4);
168 }
169
170 gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
171 vgpu->id, write_pointer, offset, status->ldw, status->udw);
172
173 if (trigger_interrupt_later)
174 return;
175
176 intel_vgpu_trigger_virtual_event(vgpu,
177 to_context_switch_event(execlist->engine));
178 }
179
emulate_execlist_ctx_schedule_out(struct intel_vgpu_execlist * execlist,struct execlist_ctx_descriptor_format * ctx)180 static int emulate_execlist_ctx_schedule_out(
181 struct intel_vgpu_execlist *execlist,
182 struct execlist_ctx_descriptor_format *ctx)
183 {
184 struct intel_vgpu *vgpu = execlist->vgpu;
185 struct intel_vgpu_execlist_slot *running = execlist->running_slot;
186 struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
187 struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0];
188 struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1];
189 struct execlist_context_status_format status;
190
191 memset(&status, 0, sizeof(status));
192
193 gvt_dbg_el("schedule out context id %x\n", ctx->context_id);
194
195 if (WARN_ON(!same_context(ctx, execlist->running_context))) {
196 gvt_vgpu_err("schedule out context is not running context,"
197 "ctx id %x running ctx id %x\n",
198 ctx->context_id,
199 execlist->running_context->context_id);
200 return -EINVAL;
201 }
202
203 /* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */
204 if (valid_context(ctx1) && same_context(ctx0, ctx)) {
205 gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n");
206
207 execlist->running_context = ctx1;
208
209 emulate_execlist_status(execlist);
210
211 status.context_complete = status.element_switch = 1;
212 status.context_id = ctx->context_id;
213
214 emulate_csb_update(execlist, &status, false);
215 /*
216 * ctx1 is not valid, ctx == ctx0
217 * ctx1 is valid, ctx1 == ctx
218 * --> last element is finished
219 * emulate:
220 * active-to-idle if there is *no* pending execlist
221 * context-complete if there *is* pending execlist
222 */
223 } else if ((!valid_context(ctx1) && same_context(ctx0, ctx))
224 || (valid_context(ctx1) && same_context(ctx1, ctx))) {
225 gvt_dbg_el("need to switch virtual execlist slot\n");
226
227 switch_virtual_execlist_slot(execlist);
228
229 emulate_execlist_status(execlist);
230
231 status.context_complete = status.active_to_idle = 1;
232 status.context_id = ctx->context_id;
233
234 if (!pending) {
235 emulate_csb_update(execlist, &status, false);
236 } else {
237 emulate_csb_update(execlist, &status, true);
238
239 memset(&status, 0, sizeof(status));
240
241 status.idle_to_active = 1;
242 status.context_id = 0;
243
244 emulate_csb_update(execlist, &status, false);
245 }
246 } else {
247 WARN_ON(1);
248 return -EINVAL;
249 }
250
251 return 0;
252 }
253
get_next_execlist_slot(struct intel_vgpu_execlist * execlist)254 static struct intel_vgpu_execlist_slot *get_next_execlist_slot(
255 struct intel_vgpu_execlist *execlist)
256 {
257 struct intel_vgpu *vgpu = execlist->vgpu;
258 u32 status_reg =
259 execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
260 struct execlist_status_format status;
261
262 status.ldw = vgpu_vreg(vgpu, status_reg);
263 status.udw = vgpu_vreg(vgpu, status_reg + 4);
264
265 if (status.execlist_queue_full) {
266 gvt_vgpu_err("virtual execlist slots are full\n");
267 return NULL;
268 }
269
270 return &execlist->slot[status.execlist_write_pointer];
271 }
272
emulate_execlist_schedule_in(struct intel_vgpu_execlist * execlist,struct execlist_ctx_descriptor_format ctx[2])273 static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
274 struct execlist_ctx_descriptor_format ctx[2])
275 {
276 struct intel_vgpu_execlist_slot *running = execlist->running_slot;
277 struct intel_vgpu_execlist_slot *slot =
278 get_next_execlist_slot(execlist);
279
280 struct execlist_ctx_descriptor_format *ctx0, *ctx1;
281 struct execlist_context_status_format status;
282 struct intel_vgpu *vgpu = execlist->vgpu;
283
284 gvt_dbg_el("emulate schedule-in\n");
285
286 if (!slot) {
287 gvt_vgpu_err("no available execlist slot\n");
288 return -EINVAL;
289 }
290
291 memset(&status, 0, sizeof(status));
292 memset(slot->ctx, 0, sizeof(slot->ctx));
293
294 slot->ctx[0] = ctx[0];
295 slot->ctx[1] = ctx[1];
296
297 gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n",
298 slot->index, ctx[0].context_id,
299 ctx[1].context_id);
300
301 /*
302 * no running execlist, make this write bundle as running execlist
303 * -> idle-to-active
304 */
305 if (!running) {
306 gvt_dbg_el("no current running execlist\n");
307
308 execlist->running_slot = slot;
309 execlist->pending_slot = NULL;
310 execlist->running_context = &slot->ctx[0];
311
312 gvt_dbg_el("running slot index %d running context %x\n",
313 execlist->running_slot->index,
314 execlist->running_context->context_id);
315
316 emulate_execlist_status(execlist);
317
318 status.idle_to_active = 1;
319 status.context_id = 0;
320
321 emulate_csb_update(execlist, &status, false);
322 return 0;
323 }
324
325 ctx0 = &running->ctx[0];
326 ctx1 = &running->ctx[1];
327
328 gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n",
329 running->index, ctx0->context_id, ctx1->context_id);
330
331 /*
332 * already has an running execlist
333 * a. running ctx1 is valid,
334 * ctx0 is finished, and running ctx1 == new execlist ctx[0]
335 * b. running ctx1 is not valid,
336 * ctx0 == new execlist ctx[0]
337 * ----> lite-restore + preempted
338 */
339 if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) &&
340 /* condition a */
341 (!same_context(ctx0, execlist->running_context))) ||
342 (!valid_context(ctx1) &&
343 same_context(ctx0, &slot->ctx[0]))) { /* condition b */
344 gvt_dbg_el("need to switch virtual execlist slot\n");
345
346 execlist->pending_slot = slot;
347 switch_virtual_execlist_slot(execlist);
348
349 emulate_execlist_status(execlist);
350
351 status.lite_restore = status.preempted = 1;
352 status.context_id = ctx[0].context_id;
353
354 emulate_csb_update(execlist, &status, false);
355 } else {
356 gvt_dbg_el("emulate as pending slot\n");
357 /*
358 * otherwise
359 * --> emulate pending execlist exist + but no preemption case
360 */
361 execlist->pending_slot = slot;
362 emulate_execlist_status(execlist);
363 }
364 return 0;
365 }
366
367 #define get_desc_from_elsp_dwords(ed, i) \
368 ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
369
prepare_execlist_workload(struct intel_vgpu_workload * workload)370 static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
371 {
372 struct intel_vgpu *vgpu = workload->vgpu;
373 struct intel_vgpu_submission *s = &vgpu->submission;
374 struct execlist_ctx_descriptor_format ctx[2];
375 int ret;
376
377 if (!workload->emulate_schedule_in)
378 return 0;
379
380 ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
381 ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
382
383 ret = emulate_execlist_schedule_in(&s->execlist[workload->engine->id],
384 ctx);
385 if (ret) {
386 gvt_vgpu_err("fail to emulate execlist schedule in\n");
387 return ret;
388 }
389 return 0;
390 }
391
complete_execlist_workload(struct intel_vgpu_workload * workload)392 static int complete_execlist_workload(struct intel_vgpu_workload *workload)
393 {
394 struct intel_vgpu *vgpu = workload->vgpu;
395 struct intel_vgpu_submission *s = &vgpu->submission;
396 struct intel_vgpu_execlist *execlist =
397 &s->execlist[workload->engine->id];
398 struct intel_vgpu_workload *next_workload;
399 struct list_head *next = workload_q_head(vgpu, workload->engine)->next;
400 bool lite_restore = false;
401 int ret = 0;
402
403 gvt_dbg_el("complete workload %p status %d\n",
404 workload, workload->status);
405
406 if (workload->status || vgpu->resetting_eng & workload->engine->mask)
407 goto out;
408
409 if (!list_empty(workload_q_head(vgpu, workload->engine))) {
410 struct execlist_ctx_descriptor_format *this_desc, *next_desc;
411
412 next_workload = container_of(next,
413 struct intel_vgpu_workload, list);
414 this_desc = &workload->ctx_desc;
415 next_desc = &next_workload->ctx_desc;
416
417 lite_restore = same_context(this_desc, next_desc);
418 }
419
420 if (lite_restore) {
421 gvt_dbg_el("next context == current - no schedule-out\n");
422 goto out;
423 }
424
425 ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
426 out:
427 return ret;
428 }
429
submit_context(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine,struct execlist_ctx_descriptor_format * desc,bool emulate_schedule_in)430 static int submit_context(struct intel_vgpu *vgpu,
431 const struct intel_engine_cs *engine,
432 struct execlist_ctx_descriptor_format *desc,
433 bool emulate_schedule_in)
434 {
435 struct intel_vgpu_submission *s = &vgpu->submission;
436 struct intel_vgpu_workload *workload = NULL;
437
438 workload = intel_vgpu_create_workload(vgpu, engine, desc);
439 if (IS_ERR(workload))
440 return PTR_ERR(workload);
441
442 workload->prepare = prepare_execlist_workload;
443 workload->complete = complete_execlist_workload;
444 workload->emulate_schedule_in = emulate_schedule_in;
445
446 if (emulate_schedule_in)
447 workload->elsp_dwords = s->execlist[engine->id].elsp_dwords;
448
449 gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
450 emulate_schedule_in);
451
452 intel_vgpu_queue_workload(workload);
453 return 0;
454 }
455
intel_vgpu_submit_execlist(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine)456 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu,
457 const struct intel_engine_cs *engine)
458 {
459 struct intel_vgpu_submission *s = &vgpu->submission;
460 struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
461 struct execlist_ctx_descriptor_format *desc[2];
462 int i, ret;
463
464 desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
465 desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
466
467 if (!desc[0]->valid) {
468 gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
469 goto inv_desc;
470 }
471
472 for (i = 0; i < ARRAY_SIZE(desc); i++) {
473 if (!desc[i]->valid)
474 continue;
475 if (!desc[i]->privilege_access) {
476 gvt_vgpu_err("unexpected GGTT elsp submission\n");
477 goto inv_desc;
478 }
479 }
480
481 /* submit workload */
482 for (i = 0; i < ARRAY_SIZE(desc); i++) {
483 if (!desc[i]->valid)
484 continue;
485 ret = submit_context(vgpu, engine, desc[i], i == 0);
486 if (ret) {
487 gvt_vgpu_err("failed to submit desc %d\n", i);
488 return ret;
489 }
490 }
491
492 return 0;
493
494 inv_desc:
495 gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
496 desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw);
497 return -EINVAL;
498 }
499
init_vgpu_execlist(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine)500 static void init_vgpu_execlist(struct intel_vgpu *vgpu,
501 const struct intel_engine_cs *engine)
502 {
503 struct intel_vgpu_submission *s = &vgpu->submission;
504 struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
505 struct execlist_context_status_pointer_format ctx_status_ptr;
506 u32 ctx_status_ptr_reg;
507
508 memset(execlist, 0, sizeof(*execlist));
509
510 execlist->vgpu = vgpu;
511 execlist->engine = engine;
512 execlist->slot[0].index = 0;
513 execlist->slot[1].index = 1;
514
515 ctx_status_ptr_reg = execlist_ring_mmio(engine, _EL_OFFSET_STATUS_PTR);
516 ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
517 ctx_status_ptr.read_ptr = 0;
518 ctx_status_ptr.write_ptr = 0x7;
519 vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
520 }
521
clean_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)522 static void clean_execlist(struct intel_vgpu *vgpu,
523 intel_engine_mask_t engine_mask)
524 {
525 struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
526 struct intel_engine_cs *engine;
527 struct intel_vgpu_submission *s = &vgpu->submission;
528 intel_engine_mask_t tmp;
529
530 for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) {
531 kfree(s->ring_scan_buffer[engine->id]);
532 s->ring_scan_buffer[engine->id] = NULL;
533 s->ring_scan_buffer_size[engine->id] = 0;
534 }
535 }
536
reset_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)537 static void reset_execlist(struct intel_vgpu *vgpu,
538 intel_engine_mask_t engine_mask)
539 {
540 struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
541 struct intel_engine_cs *engine;
542 intel_engine_mask_t tmp;
543
544 for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp)
545 init_vgpu_execlist(vgpu, engine);
546 }
547
init_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)548 static int init_execlist(struct intel_vgpu *vgpu,
549 intel_engine_mask_t engine_mask)
550 {
551 reset_execlist(vgpu, engine_mask);
552 return 0;
553 }
554
555 const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = {
556 .name = "execlist",
557 .init = init_execlist,
558 .reset = reset_execlist,
559 .clean = clean_execlist,
560 };
561