Lines Matching refs:ve
355 struct virtual_engine *ve = in need_preempt() local
356 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in need_preempt()
359 if (engine == ve->siblings[0]) { /* only preempt one sibling */ in need_preempt()
363 next = READ_ONCE(ve->request); in need_preempt()
603 struct virtual_engine *ve = container_of(ce, typeof(*ve), context); in kick_siblings() local
604 struct i915_request *next = READ_ONCE(ve->request); in kick_siblings()
607 tasklet_schedule(&ve->base.execlists.tasklet); in kick_siblings()
870 static bool virtual_matches(const struct virtual_engine *ve, in virtual_matches() argument
888 inflight = intel_context_inflight(&ve->context); in virtual_matches()
895 static void virtual_xfer_breadcrumbs(struct virtual_engine *ve, in virtual_xfer_breadcrumbs() argument
898 struct intel_engine_cs *old = ve->siblings[0]; in virtual_xfer_breadcrumbs()
903 if (!list_empty(&ve->context.signal_link)) { in virtual_xfer_breadcrumbs()
904 list_move_tail(&ve->context.signal_link, in virtual_xfer_breadcrumbs()
1051 struct virtual_engine *ve = in execlists_dequeue() local
1052 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_dequeue()
1053 struct i915_request *rq = READ_ONCE(ve->request); in execlists_dequeue()
1062 if (!virtual_matches(ve, rq, engine)) { in execlists_dequeue()
1167 struct virtual_engine *ve = in execlists_dequeue() local
1168 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_dequeue()
1171 spin_lock(&ve->base.active.lock); in execlists_dequeue()
1173 rq = ve->request; in execlists_dequeue()
1175 spin_unlock(&ve->base.active.lock); in execlists_dequeue()
1182 GEM_BUG_ON(rq != ve->request); in execlists_dequeue()
1183 GEM_BUG_ON(rq->engine != &ve->base); in execlists_dequeue()
1184 GEM_BUG_ON(rq->hw_context != &ve->context); in execlists_dequeue()
1187 if (!virtual_matches(ve, rq, engine)) { in execlists_dequeue()
1188 spin_unlock(&ve->base.active.lock); in execlists_dequeue()
1194 spin_unlock(&ve->base.active.lock); in execlists_dequeue()
1205 yesno(engine != ve->siblings[0])); in execlists_dequeue()
1207 ve->request = NULL; in execlists_dequeue()
1208 ve->base.execlists.queue_priority_hint = INT_MIN; in execlists_dequeue()
1215 if (engine != ve->siblings[0]) { in execlists_dequeue()
1216 u32 *regs = ve->context.lrc_reg_state; in execlists_dequeue()
1219 GEM_BUG_ON(READ_ONCE(ve->context.inflight)); in execlists_dequeue()
1222 if (!list_empty(&ve->context.signals)) in execlists_dequeue()
1223 virtual_xfer_breadcrumbs(ve, engine); in execlists_dequeue()
1232 for (n = 1; n < ve->num_siblings; n++) { in execlists_dequeue()
1233 if (ve->siblings[n] == engine) { in execlists_dequeue()
1234 swap(ve->siblings[n], in execlists_dequeue()
1235 ve->siblings[0]); in execlists_dequeue()
1240 GEM_BUG_ON(ve->siblings[0] != engine); in execlists_dequeue()
1257 spin_unlock(&ve->base.active.lock); in execlists_dequeue()
1263 spin_unlock(&ve->base.active.lock); in execlists_dequeue()
2613 struct virtual_engine *ve = in execlists_cancel_requests() local
2614 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in execlists_cancel_requests()
2619 spin_lock(&ve->base.active.lock); in execlists_cancel_requests()
2620 rq = fetch_and_zero(&ve->request); in execlists_cancel_requests()
2628 ve->base.execlists.queue_priority_hint = INT_MIN; in execlists_cancel_requests()
2630 spin_unlock(&ve->base.active.lock); in execlists_cancel_requests()
3412 static struct list_head *virtual_queue(struct virtual_engine *ve) in virtual_queue() argument
3414 return &ve->base.execlists.default_priolist.requests[0]; in virtual_queue()
3419 struct virtual_engine *ve = in virtual_context_destroy() local
3420 container_of(kref, typeof(*ve), context.ref); in virtual_context_destroy()
3423 GEM_BUG_ON(!list_empty(virtual_queue(ve))); in virtual_context_destroy()
3424 GEM_BUG_ON(ve->request); in virtual_context_destroy()
3425 GEM_BUG_ON(ve->context.inflight); in virtual_context_destroy()
3427 for (n = 0; n < ve->num_siblings; n++) { in virtual_context_destroy()
3428 struct intel_engine_cs *sibling = ve->siblings[n]; in virtual_context_destroy()
3429 struct rb_node *node = &ve->nodes[sibling->id].rb; in virtual_context_destroy()
3442 GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet)); in virtual_context_destroy()
3444 if (ve->context.state) in virtual_context_destroy()
3445 __execlists_context_fini(&ve->context); in virtual_context_destroy()
3446 intel_context_fini(&ve->context); in virtual_context_destroy()
3448 kfree(ve->bonds); in virtual_context_destroy()
3449 kfree(ve); in virtual_context_destroy()
3452 static void virtual_engine_initial_hint(struct virtual_engine *ve) in virtual_engine_initial_hint() argument
3469 swp = prandom_u32_max(ve->num_siblings); in virtual_engine_initial_hint()
3473 swap(ve->siblings[swp], ve->siblings[0]); in virtual_engine_initial_hint()
3474 virtual_update_register_offsets(ve->context.lrc_reg_state, in virtual_engine_initial_hint()
3475 ve->siblings[0]); in virtual_engine_initial_hint()
3480 struct virtual_engine *ve = container_of(ce, typeof(*ve), context); in virtual_context_pin() local
3484 err = __execlists_context_pin(ce, ve->siblings[0]); in virtual_context_pin()
3488 virtual_engine_initial_hint(ve); in virtual_context_pin()
3494 struct virtual_engine *ve = container_of(ce, typeof(*ve), context); in virtual_context_enter() local
3497 for (n = 0; n < ve->num_siblings; n++) in virtual_context_enter()
3498 intel_engine_pm_get(ve->siblings[n]); in virtual_context_enter()
3505 struct virtual_engine *ve = container_of(ce, typeof(*ve), context); in virtual_context_exit() local
3510 for (n = 0; n < ve->num_siblings; n++) in virtual_context_exit()
3511 intel_engine_pm_put(ve->siblings[n]); in virtual_context_exit()
3524 static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve) in virtual_submission_mask() argument
3529 rq = READ_ONCE(ve->request); in virtual_submission_mask()
3538 mask = ve->siblings[0]->mask; in virtual_submission_mask()
3542 ve->base.name, in virtual_submission_mask()
3544 mask, ve->base.execlists.queue_priority_hint); in virtual_submission_mask()
3551 struct virtual_engine * const ve = (struct virtual_engine *)data; in virtual_submission_tasklet() local
3552 const int prio = ve->base.execlists.queue_priority_hint; in virtual_submission_tasklet()
3557 mask = virtual_submission_mask(ve); in virtual_submission_tasklet()
3563 for (n = 0; READ_ONCE(ve->request) && n < ve->num_siblings; n++) { in virtual_submission_tasklet()
3564 struct intel_engine_cs *sibling = ve->siblings[n]; in virtual_submission_tasklet()
3565 struct ve_node * const node = &ve->nodes[sibling->id]; in virtual_submission_tasklet()
3631 struct virtual_engine *ve = to_virtual_engine(rq->engine); in virtual_submit_request() local
3636 ve->base.name, in virtual_submit_request()
3640 GEM_BUG_ON(ve->base.submit_request != virtual_submit_request); in virtual_submit_request()
3642 spin_lock_irqsave(&ve->base.active.lock, flags); in virtual_submit_request()
3644 old = ve->request; in virtual_submit_request()
3654 ve->base.execlists.queue_priority_hint = INT_MIN; in virtual_submit_request()
3655 ve->request = NULL; in virtual_submit_request()
3657 ve->base.execlists.queue_priority_hint = rq_prio(rq); in virtual_submit_request()
3658 ve->request = i915_request_get(rq); in virtual_submit_request()
3660 GEM_BUG_ON(!list_empty(virtual_queue(ve))); in virtual_submit_request()
3661 list_move_tail(&rq->sched.link, virtual_queue(ve)); in virtual_submit_request()
3663 tasklet_schedule(&ve->base.execlists.tasklet); in virtual_submit_request()
3666 spin_unlock_irqrestore(&ve->base.active.lock, flags); in virtual_submit_request()
3670 virtual_find_bond(struct virtual_engine *ve, in virtual_find_bond() argument
3675 for (i = 0; i < ve->num_bonds; i++) { in virtual_find_bond()
3676 if (ve->bonds[i].master == master) in virtual_find_bond()
3677 return &ve->bonds[i]; in virtual_find_bond()
3686 struct virtual_engine *ve = to_virtual_engine(rq->engine); in virtual_bond_execute() local
3692 bond = virtual_find_bond(ve, to_request(signal)->engine); in virtual_bond_execute()
3710 struct virtual_engine *ve; in intel_execlists_create_virtual() local
3720 ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL); in intel_execlists_create_virtual()
3721 if (!ve) in intel_execlists_create_virtual()
3724 ve->base.i915 = ctx->i915; in intel_execlists_create_virtual()
3725 ve->base.gt = siblings[0]->gt; in intel_execlists_create_virtual()
3726 ve->base.id = -1; in intel_execlists_create_virtual()
3728 ve->base.class = OTHER_CLASS; in intel_execlists_create_virtual()
3729 ve->base.uabi_class = I915_ENGINE_CLASS_INVALID; in intel_execlists_create_virtual()
3730 ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; in intel_execlists_create_virtual()
3731 ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; in intel_execlists_create_virtual()
3746 ve->base.saturated = ALL_ENGINES; in intel_execlists_create_virtual()
3748 snprintf(ve->base.name, sizeof(ve->base.name), "virtual"); in intel_execlists_create_virtual()
3750 intel_engine_init_active(&ve->base, ENGINE_VIRTUAL); in intel_execlists_create_virtual()
3752 intel_engine_init_execlists(&ve->base); in intel_execlists_create_virtual()
3754 ve->base.cops = &virtual_context_ops; in intel_execlists_create_virtual()
3755 ve->base.request_alloc = execlists_request_alloc; in intel_execlists_create_virtual()
3757 ve->base.schedule = i915_schedule; in intel_execlists_create_virtual()
3758 ve->base.submit_request = virtual_submit_request; in intel_execlists_create_virtual()
3759 ve->base.bond_execute = virtual_bond_execute; in intel_execlists_create_virtual()
3761 INIT_LIST_HEAD(virtual_queue(ve)); in intel_execlists_create_virtual()
3762 ve->base.execlists.queue_priority_hint = INT_MIN; in intel_execlists_create_virtual()
3763 tasklet_init(&ve->base.execlists.tasklet, in intel_execlists_create_virtual()
3765 (unsigned long)ve); in intel_execlists_create_virtual()
3767 intel_context_init(&ve->context, ctx, &ve->base); in intel_execlists_create_virtual()
3773 if (sibling->mask & ve->base.mask) { in intel_execlists_create_virtual()
3793 GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb)); in intel_execlists_create_virtual()
3794 RB_CLEAR_NODE(&ve->nodes[sibling->id].rb); in intel_execlists_create_virtual()
3796 ve->siblings[ve->num_siblings++] = sibling; in intel_execlists_create_virtual()
3797 ve->base.mask |= sibling->mask; in intel_execlists_create_virtual()
3806 if (ve->base.class != OTHER_CLASS) { in intel_execlists_create_virtual()
3807 if (ve->base.class != sibling->class) { in intel_execlists_create_virtual()
3809 sibling->class, ve->base.class); in intel_execlists_create_virtual()
3816 ve->base.class = sibling->class; in intel_execlists_create_virtual()
3817 ve->base.uabi_class = sibling->uabi_class; in intel_execlists_create_virtual()
3818 snprintf(ve->base.name, sizeof(ve->base.name), in intel_execlists_create_virtual()
3819 "v%dx%d", ve->base.class, count); in intel_execlists_create_virtual()
3820 ve->base.context_size = sibling->context_size; in intel_execlists_create_virtual()
3822 ve->base.emit_bb_start = sibling->emit_bb_start; in intel_execlists_create_virtual()
3823 ve->base.emit_flush = sibling->emit_flush; in intel_execlists_create_virtual()
3824 ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb; in intel_execlists_create_virtual()
3825 ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb; in intel_execlists_create_virtual()
3826 ve->base.emit_fini_breadcrumb_dw = in intel_execlists_create_virtual()
3829 ve->base.flags = sibling->flags; in intel_execlists_create_virtual()
3832 ve->base.flags |= I915_ENGINE_IS_VIRTUAL; in intel_execlists_create_virtual()
3834 err = __execlists_context_alloc(&ve->context, siblings[0]); in intel_execlists_create_virtual()
3838 __set_bit(CONTEXT_ALLOC_BIT, &ve->context.flags); in intel_execlists_create_virtual()
3840 return &ve->context; in intel_execlists_create_virtual()
3843 intel_context_put(&ve->context); in intel_execlists_create_virtual()
3881 struct virtual_engine *ve = to_virtual_engine(engine); in intel_virtual_engine_attach_bond() local
3886 for (n = 0; n < ve->num_siblings; n++) in intel_virtual_engine_attach_bond()
3887 if (sibling == ve->siblings[n]) in intel_virtual_engine_attach_bond()
3889 if (n == ve->num_siblings) in intel_virtual_engine_attach_bond()
3892 bond = virtual_find_bond(ve, master); in intel_virtual_engine_attach_bond()
3898 bond = krealloc(ve->bonds, in intel_virtual_engine_attach_bond()
3899 sizeof(*bond) * (ve->num_bonds + 1), in intel_virtual_engine_attach_bond()
3904 bond[ve->num_bonds].master = master; in intel_virtual_engine_attach_bond()
3905 bond[ve->num_bonds].sibling_mask = sibling->mask; in intel_virtual_engine_attach_bond()
3907 ve->bonds = bond; in intel_virtual_engine_attach_bond()
3908 ve->num_bonds++; in intel_virtual_engine_attach_bond()
3973 struct virtual_engine *ve = in intel_execlists_show_requests() local
3974 rb_entry(rb, typeof(*ve), nodes[engine->id].rb); in intel_execlists_show_requests()
3975 struct i915_request *rq = READ_ONCE(ve->request); in intel_execlists_show_requests()