Lines Matching refs:ring
37 intel_ring_initialized(struct intel_engine_cs *ring) in intel_ring_initialized() argument
39 struct drm_device *dev = ring->dev; in intel_ring_initialized()
45 struct intel_context *dctx = ring->default_context; in intel_ring_initialized()
46 struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf; in intel_ring_initialized()
50 return ring->buffer && ring->buffer->obj; in intel_ring_initialized()
67 bool intel_ring_stopped(struct intel_engine_cs *ring) in intel_ring_stopped() argument
69 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_stopped()
70 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); in intel_ring_stopped()
73 void __intel_ring_advance(struct intel_engine_cs *ring) in __intel_ring_advance() argument
75 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_advance()
77 if (intel_ring_stopped(ring)) in __intel_ring_advance()
79 ring->write_tail(ring, ringbuf->tail); in __intel_ring_advance()
83 gen2_render_ring_flush(struct intel_engine_cs *ring, in gen2_render_ring_flush() argument
97 ret = intel_ring_begin(ring, 2); in gen2_render_ring_flush()
101 intel_ring_emit(ring, cmd); in gen2_render_ring_flush()
102 intel_ring_emit(ring, MI_NOOP); in gen2_render_ring_flush()
103 intel_ring_advance(ring); in gen2_render_ring_flush()
109 gen4_render_ring_flush(struct intel_engine_cs *ring, in gen4_render_ring_flush() argument
113 struct drm_device *dev = ring->dev; in gen4_render_ring_flush()
155 ret = intel_ring_begin(ring, 2); in gen4_render_ring_flush()
159 intel_ring_emit(ring, cmd); in gen4_render_ring_flush()
160 intel_ring_emit(ring, MI_NOOP); in gen4_render_ring_flush()
161 intel_ring_advance(ring); in gen4_render_ring_flush()
204 intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) in intel_emit_post_sync_nonzero_flush() argument
206 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in intel_emit_post_sync_nonzero_flush()
210 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
214 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
215 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in intel_emit_post_sync_nonzero_flush()
217 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
218 intel_ring_emit(ring, 0); /* low dword */ in intel_emit_post_sync_nonzero_flush()
219 intel_ring_emit(ring, 0); /* high dword */ in intel_emit_post_sync_nonzero_flush()
220 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
221 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
223 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
227 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
228 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); in intel_emit_post_sync_nonzero_flush()
229 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
230 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
231 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
232 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
233 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
239 gen6_render_ring_flush(struct intel_engine_cs *ring, in gen6_render_ring_flush() argument
243 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen6_render_ring_flush()
247 ret = intel_emit_post_sync_nonzero_flush(ring); in gen6_render_ring_flush()
277 ret = intel_ring_begin(ring, 4); in gen6_render_ring_flush()
281 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen6_render_ring_flush()
282 intel_ring_emit(ring, flags); in gen6_render_ring_flush()
283 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); in gen6_render_ring_flush()
284 intel_ring_emit(ring, 0); in gen6_render_ring_flush()
285 intel_ring_advance(ring); in gen6_render_ring_flush()
291 gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) in gen7_render_ring_cs_stall_wa() argument
295 ret = intel_ring_begin(ring, 4); in gen7_render_ring_cs_stall_wa()
299 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_cs_stall_wa()
300 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in gen7_render_ring_cs_stall_wa()
302 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
303 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
304 intel_ring_advance(ring); in gen7_render_ring_cs_stall_wa()
309 static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value) in gen7_ring_fbc_flush() argument
313 if (!ring->fbc_dirty) in gen7_ring_fbc_flush()
316 ret = intel_ring_begin(ring, 6); in gen7_ring_fbc_flush()
320 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); in gen7_ring_fbc_flush()
321 intel_ring_emit(ring, MSG_FBC_REND_STATE); in gen7_ring_fbc_flush()
322 intel_ring_emit(ring, value); in gen7_ring_fbc_flush()
323 intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT); in gen7_ring_fbc_flush()
324 intel_ring_emit(ring, MSG_FBC_REND_STATE); in gen7_ring_fbc_flush()
325 intel_ring_emit(ring, ring->scratch.gtt_offset + 256); in gen7_ring_fbc_flush()
326 intel_ring_advance(ring); in gen7_ring_fbc_flush()
328 ring->fbc_dirty = false; in gen7_ring_fbc_flush()
333 gen7_render_ring_flush(struct intel_engine_cs *ring, in gen7_render_ring_flush() argument
337 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen7_render_ring_flush()
377 gen7_render_ring_cs_stall_wa(ring); in gen7_render_ring_flush()
380 ret = intel_ring_begin(ring, 4); in gen7_render_ring_flush()
384 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_flush()
385 intel_ring_emit(ring, flags); in gen7_render_ring_flush()
386 intel_ring_emit(ring, scratch_addr); in gen7_render_ring_flush()
387 intel_ring_emit(ring, 0); in gen7_render_ring_flush()
388 intel_ring_advance(ring); in gen7_render_ring_flush()
391 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE); in gen7_render_ring_flush()
397 gen8_emit_pipe_control(struct intel_engine_cs *ring, in gen8_emit_pipe_control() argument
402 ret = intel_ring_begin(ring, 6); in gen8_emit_pipe_control()
406 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); in gen8_emit_pipe_control()
407 intel_ring_emit(ring, flags); in gen8_emit_pipe_control()
408 intel_ring_emit(ring, scratch_addr); in gen8_emit_pipe_control()
409 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
410 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
411 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
412 intel_ring_advance(ring); in gen8_emit_pipe_control()
418 gen8_render_ring_flush(struct intel_engine_cs *ring, in gen8_render_ring_flush() argument
422 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen8_render_ring_flush()
442 ret = gen8_emit_pipe_control(ring, in gen8_render_ring_flush()
450 ret = gen8_emit_pipe_control(ring, flags, scratch_addr); in gen8_render_ring_flush()
455 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE); in gen8_render_ring_flush()
460 static void ring_write_tail(struct intel_engine_cs *ring, in ring_write_tail() argument
463 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_write_tail()
464 I915_WRITE_TAIL(ring, value); in ring_write_tail()
467 u64 intel_ring_get_active_head(struct intel_engine_cs *ring) in intel_ring_get_active_head() argument
469 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_get_active_head()
472 if (INTEL_INFO(ring->dev)->gen >= 8) in intel_ring_get_active_head()
473 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base), in intel_ring_get_active_head()
474 RING_ACTHD_UDW(ring->mmio_base)); in intel_ring_get_active_head()
475 else if (INTEL_INFO(ring->dev)->gen >= 4) in intel_ring_get_active_head()
476 acthd = I915_READ(RING_ACTHD(ring->mmio_base)); in intel_ring_get_active_head()
483 static void ring_setup_phys_status_page(struct intel_engine_cs *ring) in ring_setup_phys_status_page() argument
485 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_setup_phys_status_page()
489 if (INTEL_INFO(ring->dev)->gen >= 4) in ring_setup_phys_status_page()
494 static bool stop_ring(struct intel_engine_cs *ring) in stop_ring() argument
496 struct drm_i915_private *dev_priv = to_i915(ring->dev); in stop_ring()
498 if (!IS_GEN2(ring->dev)) { in stop_ring()
499 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); in stop_ring()
500 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { in stop_ring()
501 DRM_ERROR("%s : timed out trying to stop ring\n", ring->name); in stop_ring()
506 if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring)) in stop_ring()
511 I915_WRITE_CTL(ring, 0); in stop_ring()
512 I915_WRITE_HEAD(ring, 0); in stop_ring()
513 ring->write_tail(ring, 0); in stop_ring()
515 if (!IS_GEN2(ring->dev)) { in stop_ring()
516 (void)I915_READ_CTL(ring); in stop_ring()
517 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); in stop_ring()
520 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0; in stop_ring()
523 static int init_ring_common(struct intel_engine_cs *ring) in init_ring_common() argument
525 struct drm_device *dev = ring->dev; in init_ring_common()
527 struct intel_ringbuffer *ringbuf = ring->buffer; in init_ring_common()
533 if (!stop_ring(ring)) { in init_ring_common()
537 ring->name, in init_ring_common()
538 I915_READ_CTL(ring), in init_ring_common()
539 I915_READ_HEAD(ring), in init_ring_common()
540 I915_READ_TAIL(ring), in init_ring_common()
541 I915_READ_START(ring)); in init_ring_common()
543 if (!stop_ring(ring)) { in init_ring_common()
546 ring->name, in init_ring_common()
547 I915_READ_CTL(ring), in init_ring_common()
548 I915_READ_HEAD(ring), in init_ring_common()
549 I915_READ_TAIL(ring), in init_ring_common()
550 I915_READ_START(ring)); in init_ring_common()
557 intel_ring_setup_status_page(ring); in init_ring_common()
559 ring_setup_phys_status_page(ring); in init_ring_common()
562 I915_READ_HEAD(ring); in init_ring_common()
568 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
571 if (I915_READ_HEAD(ring)) in init_ring_common()
573 ring->name, I915_READ_HEAD(ring)); in init_ring_common()
574 I915_WRITE_HEAD(ring, 0); in init_ring_common()
575 (void)I915_READ_HEAD(ring); in init_ring_common()
577 I915_WRITE_CTL(ring, in init_ring_common()
582 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && in init_ring_common()
583 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && in init_ring_common()
584 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { in init_ring_common()
587 ring->name, in init_ring_common()
588 I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID, in init_ring_common()
589 I915_READ_HEAD(ring), I915_READ_TAIL(ring), in init_ring_common()
590 I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
595 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) in init_ring_common()
596 i915_kernel_lost_context(ring->dev); in init_ring_common()
598 ringbuf->head = I915_READ_HEAD(ring); in init_ring_common()
599 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; in init_ring_common()
604 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); in init_ring_common()
613 intel_fini_pipe_control(struct intel_engine_cs *ring) in intel_fini_pipe_control() argument
615 struct drm_device *dev = ring->dev; in intel_fini_pipe_control()
617 if (ring->scratch.obj == NULL) in intel_fini_pipe_control()
621 kunmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_fini_pipe_control()
622 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_fini_pipe_control()
625 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_fini_pipe_control()
626 ring->scratch.obj = NULL; in intel_fini_pipe_control()
630 intel_init_pipe_control(struct intel_engine_cs *ring) in intel_init_pipe_control() argument
634 if (ring->scratch.obj) in intel_init_pipe_control()
637 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096); in intel_init_pipe_control()
638 if (ring->scratch.obj == NULL) { in intel_init_pipe_control()
644 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); in intel_init_pipe_control()
648 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0); in intel_init_pipe_control()
652 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj); in intel_init_pipe_control()
653 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_init_pipe_control()
654 if (ring->scratch.cpu_page == NULL) { in intel_init_pipe_control()
660 ring->name, ring->scratch.gtt_offset); in intel_init_pipe_control()
664 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_init_pipe_control()
666 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_init_pipe_control()
671 static inline void intel_ring_emit_wa(struct intel_engine_cs *ring, in intel_ring_emit_wa() argument
674 struct drm_device *dev = ring->dev; in intel_ring_emit_wa()
680 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); in intel_ring_emit_wa()
681 intel_ring_emit(ring, addr); in intel_ring_emit_wa()
682 intel_ring_emit(ring, value); in intel_ring_emit_wa()
695 static int bdw_init_workarounds(struct intel_engine_cs *ring) in bdw_init_workarounds() argument
698 struct drm_device *dev = ring->dev; in bdw_init_workarounds()
713 ret = intel_ring_begin(ring, 18); in bdw_init_workarounds()
720 intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN, in bdw_init_workarounds()
725 intel_ring_emit_wa(ring, GEN7_ROW_CHICKEN2, in bdw_init_workarounds()
728 intel_ring_emit_wa(ring, HALF_SLICE_CHICKEN3, in bdw_init_workarounds()
735 intel_ring_emit_wa(ring, HDC_CHICKEN0, in bdw_init_workarounds()
739 intel_ring_emit_wa(ring, CACHE_MODE_1, in bdw_init_workarounds()
750 intel_ring_emit_wa(ring, GEN7_GT_MODE, in bdw_init_workarounds()
753 intel_ring_advance(ring); in bdw_init_workarounds()
761 static int chv_init_workarounds(struct intel_engine_cs *ring) in chv_init_workarounds() argument
764 struct drm_device *dev = ring->dev; in chv_init_workarounds()
775 ret = intel_ring_begin(ring, 12); in chv_init_workarounds()
780 intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN, in chv_init_workarounds()
784 intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN, in chv_init_workarounds()
788 intel_ring_emit_wa(ring, GEN7_ROW_CHICKEN2, in chv_init_workarounds()
792 intel_ring_emit_wa(ring, HALF_SLICE_CHICKEN3, in chv_init_workarounds()
795 intel_ring_advance(ring); in chv_init_workarounds()
800 static int init_render_ring(struct intel_engine_cs *ring) in init_render_ring() argument
802 struct drm_device *dev = ring->dev; in init_render_ring()
804 int ret = init_ring_common(ring); in init_render_ring()
834 ret = intel_init_pipe_control(ring); in init_render_ring()
853 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in init_render_ring()
858 static void render_ring_cleanup(struct intel_engine_cs *ring) in render_ring_cleanup() argument
860 struct drm_device *dev = ring->dev; in render_ring_cleanup()
869 intel_fini_pipe_control(ring); in render_ring_cleanup()
989 gen6_add_request(struct intel_engine_cs *ring) in gen6_add_request() argument
993 if (ring->semaphore.signal) in gen6_add_request()
994 ret = ring->semaphore.signal(ring, 4); in gen6_add_request()
996 ret = intel_ring_begin(ring, 4); in gen6_add_request()
1001 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in gen6_add_request()
1002 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in gen6_add_request()
1003 intel_ring_emit(ring, ring->outstanding_lazy_seqno); in gen6_add_request()
1004 intel_ring_emit(ring, MI_USER_INTERRUPT); in gen6_add_request()
1005 __intel_ring_advance(ring); in gen6_add_request()
1100 pc_render_add_request(struct intel_engine_cs *ring) in pc_render_add_request() argument
1102 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in pc_render_add_request()
1113 ret = intel_ring_begin(ring, 32); in pc_render_add_request()
1117 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1120 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1121 intel_ring_emit(ring, ring->outstanding_lazy_seqno); in pc_render_add_request()
1122 intel_ring_emit(ring, 0); in pc_render_add_request()
1123 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1125 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1127 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1129 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1131 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1133 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1135 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1139 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1140 intel_ring_emit(ring, ring->outstanding_lazy_seqno); in pc_render_add_request()
1141 intel_ring_emit(ring, 0); in pc_render_add_request()
1142 __intel_ring_advance(ring); in pc_render_add_request()
1148 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in gen6_ring_get_seqno() argument
1154 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_ring_get_seqno()
1155 POSTING_READ(RING_ACTHD(ring->mmio_base)); in gen6_ring_get_seqno()
1158 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in gen6_ring_get_seqno()
1162 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in ring_get_seqno() argument
1164 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in ring_get_seqno()
1168 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno) in ring_set_seqno() argument
1170 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); in ring_set_seqno()
1174 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in pc_render_get_seqno() argument
1176 return ring->scratch.cpu_page[0]; in pc_render_get_seqno()
1180 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno) in pc_render_set_seqno() argument
1182 ring->scratch.cpu_page[0] = seqno; in pc_render_set_seqno()
1186 gen5_ring_get_irq(struct intel_engine_cs *ring) in gen5_ring_get_irq() argument
1188 struct drm_device *dev = ring->dev; in gen5_ring_get_irq()
1196 if (ring->irq_refcount++ == 0) in gen5_ring_get_irq()
1197 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_get_irq()
1204 gen5_ring_put_irq(struct intel_engine_cs *ring) in gen5_ring_put_irq() argument
1206 struct drm_device *dev = ring->dev; in gen5_ring_put_irq()
1211 if (--ring->irq_refcount == 0) in gen5_ring_put_irq()
1212 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_put_irq()
1217 i9xx_ring_get_irq(struct intel_engine_cs *ring) in i9xx_ring_get_irq() argument
1219 struct drm_device *dev = ring->dev; in i9xx_ring_get_irq()
1227 if (ring->irq_refcount++ == 0) { in i9xx_ring_get_irq()
1228 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i9xx_ring_get_irq()
1238 i9xx_ring_put_irq(struct intel_engine_cs *ring) in i9xx_ring_put_irq() argument
1240 struct drm_device *dev = ring->dev; in i9xx_ring_put_irq()
1245 if (--ring->irq_refcount == 0) { in i9xx_ring_put_irq()
1246 dev_priv->irq_mask |= ring->irq_enable_mask; in i9xx_ring_put_irq()
1254 i8xx_ring_get_irq(struct intel_engine_cs *ring) in i8xx_ring_get_irq() argument
1256 struct drm_device *dev = ring->dev; in i8xx_ring_get_irq()
1264 if (ring->irq_refcount++ == 0) { in i8xx_ring_get_irq()
1265 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i8xx_ring_get_irq()
1275 i8xx_ring_put_irq(struct intel_engine_cs *ring) in i8xx_ring_put_irq() argument
1277 struct drm_device *dev = ring->dev; in i8xx_ring_put_irq()
1282 if (--ring->irq_refcount == 0) { in i8xx_ring_put_irq()
1283 dev_priv->irq_mask |= ring->irq_enable_mask; in i8xx_ring_put_irq()
1290 void intel_ring_setup_status_page(struct intel_engine_cs *ring) in intel_ring_setup_status_page() argument
1292 struct drm_device *dev = ring->dev; in intel_ring_setup_status_page()
1293 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_setup_status_page()
1300 switch (ring->id) { in intel_ring_setup_status_page()
1319 } else if (IS_GEN6(ring->dev)) { in intel_ring_setup_status_page()
1320 mmio = RING_HWS_PGA_GEN6(ring->mmio_base); in intel_ring_setup_status_page()
1323 mmio = RING_HWS_PGA(ring->mmio_base); in intel_ring_setup_status_page()
1326 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); in intel_ring_setup_status_page()
1337 u32 reg = RING_INSTPM(ring->mmio_base); in intel_ring_setup_status_page()
1340 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_ring_setup_status_page()
1348 ring->name); in intel_ring_setup_status_page()
1353 bsd_ring_flush(struct intel_engine_cs *ring, in bsd_ring_flush() argument
1359 ret = intel_ring_begin(ring, 2); in bsd_ring_flush()
1363 intel_ring_emit(ring, MI_FLUSH); in bsd_ring_flush()
1364 intel_ring_emit(ring, MI_NOOP); in bsd_ring_flush()
1365 intel_ring_advance(ring); in bsd_ring_flush()
1370 i9xx_add_request(struct intel_engine_cs *ring) in i9xx_add_request() argument
1374 ret = intel_ring_begin(ring, 4); in i9xx_add_request()
1378 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in i9xx_add_request()
1379 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in i9xx_add_request()
1380 intel_ring_emit(ring, ring->outstanding_lazy_seqno); in i9xx_add_request()
1381 intel_ring_emit(ring, MI_USER_INTERRUPT); in i9xx_add_request()
1382 __intel_ring_advance(ring); in i9xx_add_request()
1388 gen6_ring_get_irq(struct intel_engine_cs *ring) in gen6_ring_get_irq() argument
1390 struct drm_device *dev = ring->dev; in gen6_ring_get_irq()
1398 if (ring->irq_refcount++ == 0) { in gen6_ring_get_irq()
1399 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_get_irq()
1400 I915_WRITE_IMR(ring, in gen6_ring_get_irq()
1401 ~(ring->irq_enable_mask | in gen6_ring_get_irq()
1404 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen6_ring_get_irq()
1405 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_get_irq()
1413 gen6_ring_put_irq(struct intel_engine_cs *ring) in gen6_ring_put_irq() argument
1415 struct drm_device *dev = ring->dev; in gen6_ring_put_irq()
1420 if (--ring->irq_refcount == 0) { in gen6_ring_put_irq()
1421 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_put_irq()
1422 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in gen6_ring_put_irq()
1424 I915_WRITE_IMR(ring, ~0); in gen6_ring_put_irq()
1425 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_put_irq()
1431 hsw_vebox_get_irq(struct intel_engine_cs *ring) in hsw_vebox_get_irq() argument
1433 struct drm_device *dev = ring->dev; in hsw_vebox_get_irq()
1441 if (ring->irq_refcount++ == 0) { in hsw_vebox_get_irq()
1442 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in hsw_vebox_get_irq()
1443 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_get_irq()
1451 hsw_vebox_put_irq(struct intel_engine_cs *ring) in hsw_vebox_put_irq() argument
1453 struct drm_device *dev = ring->dev; in hsw_vebox_put_irq()
1461 if (--ring->irq_refcount == 0) { in hsw_vebox_put_irq()
1462 I915_WRITE_IMR(ring, ~0); in hsw_vebox_put_irq()
1463 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_put_irq()
1469 gen8_ring_get_irq(struct intel_engine_cs *ring) in gen8_ring_get_irq() argument
1471 struct drm_device *dev = ring->dev; in gen8_ring_get_irq()
1479 if (ring->irq_refcount++ == 0) { in gen8_ring_get_irq()
1480 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_get_irq()
1481 I915_WRITE_IMR(ring, in gen8_ring_get_irq()
1482 ~(ring->irq_enable_mask | in gen8_ring_get_irq()
1485 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen8_ring_get_irq()
1487 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_get_irq()
1495 gen8_ring_put_irq(struct intel_engine_cs *ring) in gen8_ring_put_irq() argument
1497 struct drm_device *dev = ring->dev; in gen8_ring_put_irq()
1502 if (--ring->irq_refcount == 0) { in gen8_ring_put_irq()
1503 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_put_irq()
1504 I915_WRITE_IMR(ring, in gen8_ring_put_irq()
1507 I915_WRITE_IMR(ring, ~0); in gen8_ring_put_irq()
1509 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_put_irq()
1515 i965_dispatch_execbuffer(struct intel_engine_cs *ring, in i965_dispatch_execbuffer() argument
1521 ret = intel_ring_begin(ring, 2); in i965_dispatch_execbuffer()
1525 intel_ring_emit(ring, in i965_dispatch_execbuffer()
1529 intel_ring_emit(ring, offset); in i965_dispatch_execbuffer()
1530 intel_ring_advance(ring); in i965_dispatch_execbuffer()
1540 i830_dispatch_execbuffer(struct intel_engine_cs *ring, in i830_dispatch_execbuffer() argument
1544 u32 cs_offset = ring->scratch.gtt_offset; in i830_dispatch_execbuffer()
1547 ret = intel_ring_begin(ring, 6); in i830_dispatch_execbuffer()
1552 intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1553 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096); in i830_dispatch_execbuffer()
1554 intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */ in i830_dispatch_execbuffer()
1555 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1556 intel_ring_emit(ring, 0xdeadbeef); in i830_dispatch_execbuffer()
1557 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1558 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1564 ret = intel_ring_begin(ring, 6 + 2); in i830_dispatch_execbuffer()
1572 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1573 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096); in i830_dispatch_execbuffer()
1574 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096); in i830_dispatch_execbuffer()
1575 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1576 intel_ring_emit(ring, 4096); in i830_dispatch_execbuffer()
1577 intel_ring_emit(ring, offset); in i830_dispatch_execbuffer()
1579 intel_ring_emit(ring, MI_FLUSH); in i830_dispatch_execbuffer()
1580 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1581 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1587 ret = intel_ring_begin(ring, 4); in i830_dispatch_execbuffer()
1591 intel_ring_emit(ring, MI_BATCH_BUFFER); in i830_dispatch_execbuffer()
1592 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); in i830_dispatch_execbuffer()
1593 intel_ring_emit(ring, offset + len - 8); in i830_dispatch_execbuffer()
1594 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1595 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1601 i915_dispatch_execbuffer(struct intel_engine_cs *ring, in i915_dispatch_execbuffer() argument
1607 ret = intel_ring_begin(ring, 2); in i915_dispatch_execbuffer()
1611 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); in i915_dispatch_execbuffer()
1612 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); in i915_dispatch_execbuffer()
1613 intel_ring_advance(ring); in i915_dispatch_execbuffer()
1618 static void cleanup_status_page(struct intel_engine_cs *ring) in cleanup_status_page() argument
1622 obj = ring->status_page.obj; in cleanup_status_page()
1629 ring->status_page.obj = NULL; in cleanup_status_page()
1632 static int init_status_page(struct intel_engine_cs *ring) in init_status_page() argument
1636 if ((obj = ring->status_page.obj) == NULL) { in init_status_page()
1640 obj = i915_gem_alloc_object(ring->dev, 4096); in init_status_page()
1651 if (!HAS_LLC(ring->dev)) in init_status_page()
1670 ring->status_page.obj = obj; in init_status_page()
1673 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); in init_status_page()
1674 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); in init_status_page()
1675 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_status_page()
1678 ring->name, ring->status_page.gfx_addr); in init_status_page()
1683 static int init_phys_status_page(struct intel_engine_cs *ring) in init_phys_status_page() argument
1685 struct drm_i915_private *dev_priv = ring->dev->dev_private; in init_phys_status_page()
1689 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE); in init_phys_status_page()
1694 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; in init_phys_status_page()
1695 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_phys_status_page()
1759 struct intel_engine_cs *ring) in intel_init_ring_buffer() argument
1761 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_init_ring_buffer()
1768 ring->buffer = ringbuf; in intel_init_ring_buffer()
1771 ring->dev = dev; in intel_init_ring_buffer()
1772 INIT_LIST_HEAD(&ring->active_list); in intel_init_ring_buffer()
1773 INIT_LIST_HEAD(&ring->request_list); in intel_init_ring_buffer()
1774 INIT_LIST_HEAD(&ring->execlist_queue); in intel_init_ring_buffer()
1776 ringbuf->ring = ring; in intel_init_ring_buffer()
1777 memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); in intel_init_ring_buffer()
1779 init_waitqueue_head(&ring->irq_queue); in intel_init_ring_buffer()
1782 ret = init_status_page(ring); in intel_init_ring_buffer()
1786 BUG_ON(ring->id != RCS); in intel_init_ring_buffer()
1787 ret = init_phys_status_page(ring); in intel_init_ring_buffer()
1794 DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret); in intel_init_ring_buffer()
1806 ret = i915_cmd_parser_init_ring(ring); in intel_init_ring_buffer()
1810 ret = ring->init(ring); in intel_init_ring_buffer()
1818 ring->buffer = NULL; in intel_init_ring_buffer()
1822 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) in intel_cleanup_ring_buffer() argument
1824 struct drm_i915_private *dev_priv = to_i915(ring->dev); in intel_cleanup_ring_buffer()
1825 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_cleanup_ring_buffer()
1827 if (!intel_ring_initialized(ring)) in intel_cleanup_ring_buffer()
1830 intel_stop_ring_buffer(ring); in intel_cleanup_ring_buffer()
1831 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_cleanup_ring_buffer()
1834 ring->preallocated_lazy_request = NULL; in intel_cleanup_ring_buffer()
1835 ring->outstanding_lazy_seqno = 0; in intel_cleanup_ring_buffer()
1837 if (ring->cleanup) in intel_cleanup_ring_buffer()
1838 ring->cleanup(ring); in intel_cleanup_ring_buffer()
1840 cleanup_status_page(ring); in intel_cleanup_ring_buffer()
1842 i915_cmd_parser_fini_ring(ring); in intel_cleanup_ring_buffer()
1845 ring->buffer = NULL; in intel_cleanup_ring_buffer()
1848 static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) in intel_ring_wait_request() argument
1850 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_ring_wait_request()
1864 list_for_each_entry(request, &ring->request_list, list) { in intel_ring_wait_request()
1875 ret = i915_wait_seqno(ring, seqno); in intel_ring_wait_request()
1879 i915_gem_retire_requests_ring(ring); in intel_ring_wait_request()
1887 static int ring_wait_for_space(struct intel_engine_cs *ring, int n) in ring_wait_for_space() argument
1889 struct drm_device *dev = ring->dev; in ring_wait_for_space()
1891 struct intel_ringbuffer *ringbuf = ring->buffer; in ring_wait_for_space()
1895 ret = intel_ring_wait_request(ring, n); in ring_wait_for_space()
1900 __intel_ring_advance(ring); in ring_wait_for_space()
1909 trace_i915_ring_wait_begin(ring); in ring_wait_for_space()
1911 ringbuf->head = I915_READ_HEAD(ring); in ring_wait_for_space()
1942 trace_i915_ring_wait_end(ring); in ring_wait_for_space()
1946 static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) in intel_wrap_ring_buffer() argument
1949 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_wrap_ring_buffer()
1953 int ret = ring_wait_for_space(ring, rem); in intel_wrap_ring_buffer()
1969 int intel_ring_idle(struct intel_engine_cs *ring) in intel_ring_idle() argument
1975 if (ring->outstanding_lazy_seqno) { in intel_ring_idle()
1976 ret = i915_add_request(ring, NULL); in intel_ring_idle()
1982 if (list_empty(&ring->request_list)) in intel_ring_idle()
1985 seqno = list_entry(ring->request_list.prev, in intel_ring_idle()
1989 return i915_wait_seqno(ring, seqno); in intel_ring_idle()
1993 intel_ring_alloc_seqno(struct intel_engine_cs *ring) in intel_ring_alloc_seqno() argument
1995 if (ring->outstanding_lazy_seqno) in intel_ring_alloc_seqno()
1998 if (ring->preallocated_lazy_request == NULL) { in intel_ring_alloc_seqno()
2005 ring->preallocated_lazy_request = request; in intel_ring_alloc_seqno()
2008 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno); in intel_ring_alloc_seqno()
2011 static int __intel_ring_prepare(struct intel_engine_cs *ring, in __intel_ring_prepare() argument
2014 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_prepare()
2018 ret = intel_wrap_ring_buffer(ring); in __intel_ring_prepare()
2024 ret = ring_wait_for_space(ring, bytes); in __intel_ring_prepare()
2032 int intel_ring_begin(struct intel_engine_cs *ring, in intel_ring_begin() argument
2035 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_begin()
2043 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); in intel_ring_begin()
2048 ret = intel_ring_alloc_seqno(ring); in intel_ring_begin()
2052 ring->buffer->space -= num_dwords * sizeof(uint32_t); in intel_ring_begin()
2057 int intel_ring_cacheline_align(struct intel_engine_cs *ring) in intel_ring_cacheline_align() argument
2059 int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); in intel_ring_cacheline_align()
2066 ret = intel_ring_begin(ring, num_dwords); in intel_ring_cacheline_align()
2071 intel_ring_emit(ring, MI_NOOP); in intel_ring_cacheline_align()
2073 intel_ring_advance(ring); in intel_ring_cacheline_align()
2078 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) in intel_ring_init_seqno() argument
2080 struct drm_device *dev = ring->dev; in intel_ring_init_seqno()
2083 BUG_ON(ring->outstanding_lazy_seqno); in intel_ring_init_seqno()
2086 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); in intel_ring_init_seqno()
2087 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); in intel_ring_init_seqno()
2089 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0); in intel_ring_init_seqno()
2092 ring->set_seqno(ring, seqno); in intel_ring_init_seqno()
2093 ring->hangcheck.seqno = seqno; in intel_ring_init_seqno()
2096 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, in gen6_bsd_ring_write_tail() argument
2099 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_bsd_ring_write_tail()
2119 I915_WRITE_TAIL(ring, value); in gen6_bsd_ring_write_tail()
2120 POSTING_READ(RING_TAIL(ring->mmio_base)); in gen6_bsd_ring_write_tail()
2129 static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, in gen6_bsd_ring_flush() argument
2135 ret = intel_ring_begin(ring, 4); in gen6_bsd_ring_flush()
2140 if (INTEL_INFO(ring->dev)->gen >= 8) in gen6_bsd_ring_flush()
2159 intel_ring_emit(ring, cmd); in gen6_bsd_ring_flush()
2160 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_bsd_ring_flush()
2161 if (INTEL_INFO(ring->dev)->gen >= 8) { in gen6_bsd_ring_flush()
2162 intel_ring_emit(ring, 0); /* upper addr */ in gen6_bsd_ring_flush()
2163 intel_ring_emit(ring, 0); /* value */ in gen6_bsd_ring_flush()
2165 intel_ring_emit(ring, 0); in gen6_bsd_ring_flush()
2166 intel_ring_emit(ring, MI_NOOP); in gen6_bsd_ring_flush()
2168 intel_ring_advance(ring); in gen6_bsd_ring_flush()
2173 gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in gen8_ring_dispatch_execbuffer() argument
2177 bool ppgtt = USES_PPGTT(ring->dev) && !(flags & I915_DISPATCH_SECURE); in gen8_ring_dispatch_execbuffer()
2180 ret = intel_ring_begin(ring, 4); in gen8_ring_dispatch_execbuffer()
2185 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); in gen8_ring_dispatch_execbuffer()
2186 intel_ring_emit(ring, lower_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2187 intel_ring_emit(ring, upper_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2188 intel_ring_emit(ring, MI_NOOP); in gen8_ring_dispatch_execbuffer()
2189 intel_ring_advance(ring); in gen8_ring_dispatch_execbuffer()
2195 hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in hsw_ring_dispatch_execbuffer() argument
2201 ret = intel_ring_begin(ring, 2); in hsw_ring_dispatch_execbuffer()
2205 intel_ring_emit(ring, in hsw_ring_dispatch_execbuffer()
2210 intel_ring_emit(ring, offset); in hsw_ring_dispatch_execbuffer()
2211 intel_ring_advance(ring); in hsw_ring_dispatch_execbuffer()
2217 gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in gen6_ring_dispatch_execbuffer() argument
2223 ret = intel_ring_begin(ring, 2); in gen6_ring_dispatch_execbuffer()
2227 intel_ring_emit(ring, in gen6_ring_dispatch_execbuffer()
2231 intel_ring_emit(ring, offset); in gen6_ring_dispatch_execbuffer()
2232 intel_ring_advance(ring); in gen6_ring_dispatch_execbuffer()
2239 static int gen6_ring_flush(struct intel_engine_cs *ring, in gen6_ring_flush() argument
2242 struct drm_device *dev = ring->dev; in gen6_ring_flush()
2246 ret = intel_ring_begin(ring, 4); in gen6_ring_flush()
2251 if (INTEL_INFO(ring->dev)->gen >= 8) in gen6_ring_flush()
2269 intel_ring_emit(ring, cmd); in gen6_ring_flush()
2270 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_ring_flush()
2271 if (INTEL_INFO(ring->dev)->gen >= 8) { in gen6_ring_flush()
2272 intel_ring_emit(ring, 0); /* upper addr */ in gen6_ring_flush()
2273 intel_ring_emit(ring, 0); /* value */ in gen6_ring_flush()
2275 intel_ring_emit(ring, 0); in gen6_ring_flush()
2276 intel_ring_emit(ring, MI_NOOP); in gen6_ring_flush()
2278 intel_ring_advance(ring); in gen6_ring_flush()
2281 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN); in gen6_ring_flush()
2289 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; in intel_init_render_ring_buffer() local
2293 ring->name = "render ring"; in intel_init_render_ring_buffer()
2294 ring->id = RCS; in intel_init_render_ring_buffer()
2295 ring->mmio_base = RENDER_RING_BASE; in intel_init_render_ring_buffer()
2315 ring->init_context = chv_init_workarounds; in intel_init_render_ring_buffer()
2317 ring->init_context = bdw_init_workarounds; in intel_init_render_ring_buffer()
2318 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2319 ring->flush = gen8_render_ring_flush; in intel_init_render_ring_buffer()
2320 ring->irq_get = gen8_ring_get_irq; in intel_init_render_ring_buffer()
2321 ring->irq_put = gen8_ring_put_irq; in intel_init_render_ring_buffer()
2322 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2323 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2324 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2327 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_render_ring_buffer()
2328 ring->semaphore.signal = gen8_rcs_signal; in intel_init_render_ring_buffer()
2332 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2333 ring->flush = gen7_render_ring_flush; in intel_init_render_ring_buffer()
2335 ring->flush = gen6_render_ring_flush; in intel_init_render_ring_buffer()
2336 ring->irq_get = gen6_ring_get_irq; in intel_init_render_ring_buffer()
2337 ring->irq_put = gen6_ring_put_irq; in intel_init_render_ring_buffer()
2338 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2339 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2340 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2342 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_render_ring_buffer()
2343 ring->semaphore.signal = gen6_signal; in intel_init_render_ring_buffer()
2351 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2352 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; in intel_init_render_ring_buffer()
2353 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; in intel_init_render_ring_buffer()
2354 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; in intel_init_render_ring_buffer()
2355 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2356 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2357 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; in intel_init_render_ring_buffer()
2358 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; in intel_init_render_ring_buffer()
2359 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; in intel_init_render_ring_buffer()
2360 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2363 ring->add_request = pc_render_add_request; in intel_init_render_ring_buffer()
2364 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2365 ring->get_seqno = pc_render_get_seqno; in intel_init_render_ring_buffer()
2366 ring->set_seqno = pc_render_set_seqno; in intel_init_render_ring_buffer()
2367 ring->irq_get = gen5_ring_get_irq; in intel_init_render_ring_buffer()
2368 ring->irq_put = gen5_ring_put_irq; in intel_init_render_ring_buffer()
2369 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT | in intel_init_render_ring_buffer()
2372 ring->add_request = i9xx_add_request; in intel_init_render_ring_buffer()
2374 ring->flush = gen2_render_ring_flush; in intel_init_render_ring_buffer()
2376 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2377 ring->get_seqno = ring_get_seqno; in intel_init_render_ring_buffer()
2378 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2380 ring->irq_get = i8xx_ring_get_irq; in intel_init_render_ring_buffer()
2381 ring->irq_put = i8xx_ring_put_irq; in intel_init_render_ring_buffer()
2383 ring->irq_get = i9xx_ring_get_irq; in intel_init_render_ring_buffer()
2384 ring->irq_put = i9xx_ring_put_irq; in intel_init_render_ring_buffer()
2386 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_init_render_ring_buffer()
2388 ring->write_tail = ring_write_tail; in intel_init_render_ring_buffer()
2391 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2393 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2395 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2397 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_render_ring_buffer()
2399 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_init_render_ring_buffer()
2401 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_init_render_ring_buffer()
2402 ring->init = init_render_ring; in intel_init_render_ring_buffer()
2403 ring->cleanup = render_ring_cleanup; in intel_init_render_ring_buffer()
2420 ring->scratch.obj = obj; in intel_init_render_ring_buffer()
2421 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); in intel_init_render_ring_buffer()
2424 return intel_init_ring_buffer(dev, ring); in intel_init_render_ring_buffer()
2430 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; in intel_render_ring_init_dri() local
2431 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_render_ring_init_dri()
2438 ring->buffer = ringbuf; in intel_render_ring_init_dri()
2441 ring->name = "render ring"; in intel_render_ring_init_dri()
2442 ring->id = RCS; in intel_render_ring_init_dri()
2443 ring->mmio_base = RENDER_RING_BASE; in intel_render_ring_init_dri()
2454 ring->add_request = i9xx_add_request; in intel_render_ring_init_dri()
2456 ring->flush = gen2_render_ring_flush; in intel_render_ring_init_dri()
2458 ring->flush = gen4_render_ring_flush; in intel_render_ring_init_dri()
2459 ring->get_seqno = ring_get_seqno; in intel_render_ring_init_dri()
2460 ring->set_seqno = ring_set_seqno; in intel_render_ring_init_dri()
2462 ring->irq_get = i8xx_ring_get_irq; in intel_render_ring_init_dri()
2463 ring->irq_put = i8xx_ring_put_irq; in intel_render_ring_init_dri()
2465 ring->irq_get = i9xx_ring_get_irq; in intel_render_ring_init_dri()
2466 ring->irq_put = i9xx_ring_put_irq; in intel_render_ring_init_dri()
2468 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_render_ring_init_dri()
2469 ring->write_tail = ring_write_tail; in intel_render_ring_init_dri()
2471 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_render_ring_init_dri()
2473 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_render_ring_init_dri()
2475 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_render_ring_init_dri()
2476 ring->init = init_render_ring; in intel_render_ring_init_dri()
2477 ring->cleanup = render_ring_cleanup; in intel_render_ring_init_dri()
2479 ring->dev = dev; in intel_render_ring_init_dri()
2480 INIT_LIST_HEAD(&ring->active_list); in intel_render_ring_init_dri()
2481 INIT_LIST_HEAD(&ring->request_list); in intel_render_ring_init_dri()
2485 if (IS_I830(ring->dev) || IS_845G(ring->dev)) in intel_render_ring_init_dri()
2497 ret = init_phys_status_page(ring); in intel_render_ring_init_dri()
2508 ring->buffer = NULL; in intel_render_ring_init_dri()
2515 struct intel_engine_cs *ring = &dev_priv->ring[VCS]; in intel_init_bsd_ring_buffer() local
2517 ring->name = "bsd ring"; in intel_init_bsd_ring_buffer()
2518 ring->id = VCS; in intel_init_bsd_ring_buffer()
2520 ring->write_tail = ring_write_tail; in intel_init_bsd_ring_buffer()
2522 ring->mmio_base = GEN6_BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2525 ring->write_tail = gen6_bsd_ring_write_tail; in intel_init_bsd_ring_buffer()
2526 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd_ring_buffer()
2527 ring->add_request = gen6_add_request; in intel_init_bsd_ring_buffer()
2528 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd_ring_buffer()
2529 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2531 ring->irq_enable_mask = in intel_init_bsd_ring_buffer()
2533 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd_ring_buffer()
2534 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd_ring_buffer()
2535 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2538 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd_ring_buffer()
2539 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd_ring_buffer()
2543 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2544 ring->irq_get = gen6_ring_get_irq; in intel_init_bsd_ring_buffer()
2545 ring->irq_put = gen6_ring_put_irq; in intel_init_bsd_ring_buffer()
2546 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2549 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_bsd_ring_buffer()
2550 ring->semaphore.signal = gen6_signal; in intel_init_bsd_ring_buffer()
2551 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; in intel_init_bsd_ring_buffer()
2552 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2553 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; in intel_init_bsd_ring_buffer()
2554 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; in intel_init_bsd_ring_buffer()
2555 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2556 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; in intel_init_bsd_ring_buffer()
2557 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2558 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; in intel_init_bsd_ring_buffer()
2559 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; in intel_init_bsd_ring_buffer()
2560 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2564 ring->mmio_base = BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2565 ring->flush = bsd_ring_flush; in intel_init_bsd_ring_buffer()
2566 ring->add_request = i9xx_add_request; in intel_init_bsd_ring_buffer()
2567 ring->get_seqno = ring_get_seqno; in intel_init_bsd_ring_buffer()
2568 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2570 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2571 ring->irq_get = gen5_ring_get_irq; in intel_init_bsd_ring_buffer()
2572 ring->irq_put = gen5_ring_put_irq; in intel_init_bsd_ring_buffer()
2574 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2575 ring->irq_get = i9xx_ring_get_irq; in intel_init_bsd_ring_buffer()
2576 ring->irq_put = i9xx_ring_put_irq; in intel_init_bsd_ring_buffer()
2578 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_bsd_ring_buffer()
2580 ring->init = init_ring_common; in intel_init_bsd_ring_buffer()
2582 return intel_init_ring_buffer(dev, ring); in intel_init_bsd_ring_buffer()
2592 struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; in intel_init_bsd2_ring_buffer() local
2599 ring->name = "bsd2 ring"; in intel_init_bsd2_ring_buffer()
2600 ring->id = VCS2; in intel_init_bsd2_ring_buffer()
2602 ring->write_tail = ring_write_tail; in intel_init_bsd2_ring_buffer()
2603 ring->mmio_base = GEN8_BSD2_RING_BASE; in intel_init_bsd2_ring_buffer()
2604 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd2_ring_buffer()
2605 ring->add_request = gen6_add_request; in intel_init_bsd2_ring_buffer()
2606 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd2_ring_buffer()
2607 ring->set_seqno = ring_set_seqno; in intel_init_bsd2_ring_buffer()
2608 ring->irq_enable_mask = in intel_init_bsd2_ring_buffer()
2610 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd2_ring_buffer()
2611 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd2_ring_buffer()
2612 ring->dispatch_execbuffer = in intel_init_bsd2_ring_buffer()
2615 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd2_ring_buffer()
2616 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd2_ring_buffer()
2619 ring->init = init_ring_common; in intel_init_bsd2_ring_buffer()
2621 return intel_init_ring_buffer(dev, ring); in intel_init_bsd2_ring_buffer()
2627 struct intel_engine_cs *ring = &dev_priv->ring[BCS]; in intel_init_blt_ring_buffer() local
2629 ring->name = "blitter ring"; in intel_init_blt_ring_buffer()
2630 ring->id = BCS; in intel_init_blt_ring_buffer()
2632 ring->mmio_base = BLT_RING_BASE; in intel_init_blt_ring_buffer()
2633 ring->write_tail = ring_write_tail; in intel_init_blt_ring_buffer()
2634 ring->flush = gen6_ring_flush; in intel_init_blt_ring_buffer()
2635 ring->add_request = gen6_add_request; in intel_init_blt_ring_buffer()
2636 ring->get_seqno = gen6_ring_get_seqno; in intel_init_blt_ring_buffer()
2637 ring->set_seqno = ring_set_seqno; in intel_init_blt_ring_buffer()
2639 ring->irq_enable_mask = in intel_init_blt_ring_buffer()
2641 ring->irq_get = gen8_ring_get_irq; in intel_init_blt_ring_buffer()
2642 ring->irq_put = gen8_ring_put_irq; in intel_init_blt_ring_buffer()
2643 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2645 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_blt_ring_buffer()
2646 ring->semaphore.signal = gen8_xcs_signal; in intel_init_blt_ring_buffer()
2650 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; in intel_init_blt_ring_buffer()
2651 ring->irq_get = gen6_ring_get_irq; in intel_init_blt_ring_buffer()
2652 ring->irq_put = gen6_ring_put_irq; in intel_init_blt_ring_buffer()
2653 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2655 ring->semaphore.signal = gen6_signal; in intel_init_blt_ring_buffer()
2656 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_blt_ring_buffer()
2664 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; in intel_init_blt_ring_buffer()
2665 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; in intel_init_blt_ring_buffer()
2666 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2667 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; in intel_init_blt_ring_buffer()
2668 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2669 ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; in intel_init_blt_ring_buffer()
2670 ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; in intel_init_blt_ring_buffer()
2671 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2672 ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; in intel_init_blt_ring_buffer()
2673 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2676 ring->init = init_ring_common; in intel_init_blt_ring_buffer()
2678 return intel_init_ring_buffer(dev, ring); in intel_init_blt_ring_buffer()
2684 struct intel_engine_cs *ring = &dev_priv->ring[VECS]; in intel_init_vebox_ring_buffer() local
2686 ring->name = "video enhancement ring"; in intel_init_vebox_ring_buffer()
2687 ring->id = VECS; in intel_init_vebox_ring_buffer()
2689 ring->mmio_base = VEBOX_RING_BASE; in intel_init_vebox_ring_buffer()
2690 ring->write_tail = ring_write_tail; in intel_init_vebox_ring_buffer()
2691 ring->flush = gen6_ring_flush; in intel_init_vebox_ring_buffer()
2692 ring->add_request = gen6_add_request; in intel_init_vebox_ring_buffer()
2693 ring->get_seqno = gen6_ring_get_seqno; in intel_init_vebox_ring_buffer()
2694 ring->set_seqno = ring_set_seqno; in intel_init_vebox_ring_buffer()
2697 ring->irq_enable_mask = in intel_init_vebox_ring_buffer()
2699 ring->irq_get = gen8_ring_get_irq; in intel_init_vebox_ring_buffer()
2700 ring->irq_put = gen8_ring_put_irq; in intel_init_vebox_ring_buffer()
2701 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2703 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_vebox_ring_buffer()
2704 ring->semaphore.signal = gen8_xcs_signal; in intel_init_vebox_ring_buffer()
2708 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; in intel_init_vebox_ring_buffer()
2709 ring->irq_get = hsw_vebox_get_irq; in intel_init_vebox_ring_buffer()
2710 ring->irq_put = hsw_vebox_put_irq; in intel_init_vebox_ring_buffer()
2711 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2713 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_vebox_ring_buffer()
2714 ring->semaphore.signal = gen6_signal; in intel_init_vebox_ring_buffer()
2715 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; in intel_init_vebox_ring_buffer()
2716 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; in intel_init_vebox_ring_buffer()
2717 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; in intel_init_vebox_ring_buffer()
2718 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2719 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2720 ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; in intel_init_vebox_ring_buffer()
2721 ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; in intel_init_vebox_ring_buffer()
2722 ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; in intel_init_vebox_ring_buffer()
2723 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
2724 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
2727 ring->init = init_ring_common; in intel_init_vebox_ring_buffer()
2729 return intel_init_ring_buffer(dev, ring); in intel_init_vebox_ring_buffer()
2733 intel_ring_flush_all_caches(struct intel_engine_cs *ring) in intel_ring_flush_all_caches() argument
2737 if (!ring->gpu_caches_dirty) in intel_ring_flush_all_caches()
2740 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
2744 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
2746 ring->gpu_caches_dirty = false; in intel_ring_flush_all_caches()
2751 intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) in intel_ring_invalidate_all_caches() argument
2757 if (ring->gpu_caches_dirty) in intel_ring_invalidate_all_caches()
2760 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
2764 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
2766 ring->gpu_caches_dirty = false; in intel_ring_invalidate_all_caches()
2771 intel_stop_ring_buffer(struct intel_engine_cs *ring) in intel_stop_ring_buffer() argument
2775 if (!intel_ring_initialized(ring)) in intel_stop_ring_buffer()
2778 ret = intel_ring_idle(ring); in intel_stop_ring_buffer()
2779 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) in intel_stop_ring_buffer()
2781 ring->name, ret); in intel_stop_ring_buffer()
2783 stop_ring(ring); in intel_stop_ring_buffer()