54 gen6_emit_post_sync_nonzero_flush(struct i915_request *rq)  in gen6_emit_post_sync_nonzero_flush()  argument
57 		intel_gt_scratch_offset(rq->engine->gt,  in gen6_emit_post_sync_nonzero_flush()
61 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
71 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
73 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
83 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
88 int gen6_emit_flush_rcs(struct i915_request *rq, u32 mode)  in gen6_emit_flush_rcs()  argument
91 		intel_gt_scratch_offset(rq->engine->gt,  in gen6_emit_flush_rcs()
97 	ret = gen6_emit_post_sync_nonzero_flush(rq);  in gen6_emit_flush_rcs()
129 	cs = intel_ring_begin(rq, 4);  in gen6_emit_flush_rcs()
137 	intel_ring_advance(rq, cs);  in gen6_emit_flush_rcs()
142 u32 *gen6_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen6_emit_breadcrumb_rcs()  argument
152 	*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen6_emit_breadcrumb_rcs()
164 	*cs++ = i915_request_active_seqno(rq) |  in gen6_emit_breadcrumb_rcs()
166 	*cs++ = rq->fence.seqno;  in gen6_emit_breadcrumb_rcs()
171 	rq->tail = intel_ring_offset(rq, cs);  in gen6_emit_breadcrumb_rcs()
172 	assert_ring_tail_valid(rq->ring, rq->tail);  in gen6_emit_breadcrumb_rcs()
177 static int mi_flush_dw(struct i915_request *rq, u32 flags)  in mi_flush_dw()  argument
181 	cs = intel_ring_begin(rq, 4);  in mi_flush_dw()
208 	intel_ring_advance(rq, cs);  in mi_flush_dw()
213 static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags)  in gen6_flush_dw()  argument
215 	return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0);  in gen6_flush_dw()
218 int gen6_emit_flush_xcs(struct i915_request *rq, u32 mode)  in gen6_emit_flush_xcs()  argument
220 	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB);  in gen6_emit_flush_xcs()
223 int gen6_emit_flush_vcs(struct i915_request *rq, u32 mode)  in gen6_emit_flush_vcs()  argument
225 	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD);  in gen6_emit_flush_vcs()
228 int gen6_emit_bb_start(struct i915_request *rq,  in gen6_emit_bb_start()  argument
239 	cs = intel_ring_begin(rq, 2);  in gen6_emit_bb_start()
244 	intel_ring_advance(rq, cs);  in gen6_emit_bb_start()
250 hsw_emit_bb_start(struct i915_request *rq,  in hsw_emit_bb_start()  argument
261 	cs = intel_ring_begin(rq, 2);  in hsw_emit_bb_start()
266 	intel_ring_advance(rq, cs);  in hsw_emit_bb_start()
271 static int gen7_stall_cs(struct i915_request *rq)  in gen7_stall_cs()  argument
275 	cs = intel_ring_begin(rq, 4);  in gen7_stall_cs()
283 	intel_ring_advance(rq, cs);  in gen7_stall_cs()
288 int gen7_emit_flush_rcs(struct i915_request *rq, u32 mode)  in gen7_emit_flush_rcs()  argument
291 		intel_gt_scratch_offset(rq->engine->gt,  in gen7_emit_flush_rcs()
336 		gen7_stall_cs(rq);  in gen7_emit_flush_rcs()
339 	cs = intel_ring_begin(rq, 4);  in gen7_emit_flush_rcs()
347 	intel_ring_advance(rq, cs);  in gen7_emit_flush_rcs()
352 u32 *gen7_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen7_emit_breadcrumb_rcs()  argument
362 	*cs++ = i915_request_active_seqno(rq);  in gen7_emit_breadcrumb_rcs()
363 	*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_rcs()
368 	rq->tail = intel_ring_offset(rq, cs);  in gen7_emit_breadcrumb_rcs()
369 	assert_ring_tail_valid(rq->ring, rq->tail);  in gen7_emit_breadcrumb_rcs()
374 u32 *gen6_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen6_emit_breadcrumb_xcs()  argument
376 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);  in gen6_emit_breadcrumb_xcs()
377 	GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);  in gen6_emit_breadcrumb_xcs()
381 	*cs++ = rq->fence.seqno;  in gen6_emit_breadcrumb_xcs()
385 	rq->tail = intel_ring_offset(rq, cs);  in gen6_emit_breadcrumb_xcs()
386 	assert_ring_tail_valid(rq->ring, rq->tail);  in gen6_emit_breadcrumb_xcs()
392 u32 *gen7_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen7_emit_breadcrumb_xcs()  argument
396 	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);  in gen7_emit_breadcrumb_xcs()
397 	GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);  in gen7_emit_breadcrumb_xcs()
402 	*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_xcs()
407 		*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_xcs()
417 	rq->tail = intel_ring_offset(rq, cs);  in gen7_emit_breadcrumb_xcs()
418 	assert_ring_tail_valid(rq->ring, rq->tail);  in gen7_emit_breadcrumb_xcs()