Lines Matching refs:stream
414 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream) in gen12_oa_hw_tail_read() argument
416 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
422 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream) in gen8_oa_hw_tail_read() argument
424 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
429 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) in gen7_oa_hw_tail_read() argument
431 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
461 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) in oa_buffer_check_unlocked() argument
463 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
464 int report_size = stream->oa_buffer.format_size; in oa_buffer_check_unlocked()
474 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
476 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
485 if (hw_tail == stream->oa_buffer.aging_tail && in oa_buffer_check_unlocked()
486 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { in oa_buffer_check_unlocked()
491 stream->oa_buffer.tail = stream->oa_buffer.aging_tail; in oa_buffer_check_unlocked()
499 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
500 aged_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
517 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); in oa_buffer_check_unlocked()
526 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
531 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
532 stream->oa_buffer.aging_tail = gtt_offset + hw_tail; in oa_buffer_check_unlocked()
533 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
536 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, in oa_buffer_check_unlocked()
537 stream->oa_buffer.head - gtt_offset) >= report_size; in oa_buffer_check_unlocked()
539 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
559 static int append_oa_status(struct i915_perf_stream *stream, in append_oa_status() argument
595 static int append_oa_sample(struct i915_perf_stream *stream, in append_oa_sample() argument
601 int report_size = stream->oa_buffer.format_size; in append_oa_sample()
606 header.size = stream->sample_size; in append_oa_sample()
644 static int gen8_append_oa_reports(struct i915_perf_stream *stream, in gen8_append_oa_reports() argument
649 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
650 int report_size = stream->oa_buffer.format_size; in gen8_append_oa_reports()
651 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
652 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
660 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
663 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
665 head = stream->oa_buffer.head; in gen8_append_oa_reports()
666 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
668 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
726 (IS_GEN(stream->perf->i915, 12) ? in gen8_append_oa_reports()
730 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen8_append_oa_reports()
735 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
745 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && in gen8_append_oa_reports()
746 INTEL_GEN(stream->perf->i915) <= 11) in gen8_append_oa_reports()
780 if (!stream->perf->exclusive_stream->ctx || in gen8_append_oa_reports()
781 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
782 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
789 if (stream->perf->exclusive_stream->ctx && in gen8_append_oa_reports()
790 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
794 ret = append_oa_sample(stream, buf, count, offset, in gen8_append_oa_reports()
799 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
813 oaheadptr = IS_GEN(stream->perf->i915, 12) ? in gen8_append_oa_reports()
816 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
825 stream->oa_buffer.head = head; in gen8_append_oa_reports()
827 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
853 static int gen8_oa_read(struct i915_perf_stream *stream, in gen8_oa_read() argument
858 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
863 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
866 oastatus_reg = IS_GEN(stream->perf->i915, 12) ? in gen8_oa_read()
886 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
892 stream->period_exponent); in gen8_oa_read()
894 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
895 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
905 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
918 return gen8_append_oa_reports(stream, buf, count, offset); in gen8_oa_read()
941 static int gen7_append_oa_reports(struct i915_perf_stream *stream, in gen7_append_oa_reports() argument
946 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
947 int report_size = stream->oa_buffer.format_size; in gen7_append_oa_reports()
948 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
949 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
957 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
960 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
962 head = stream->oa_buffer.head; in gen7_append_oa_reports()
963 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
965 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1015 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1020 ret = append_oa_sample(stream, buf, count, offset, report); in gen7_append_oa_reports()
1032 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1042 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1044 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1066 static int gen7_oa_read(struct i915_perf_stream *stream, in gen7_oa_read() argument
1071 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1075 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1085 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1108 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1114 stream->period_exponent); in gen7_oa_read()
1116 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1117 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1123 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1127 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1131 return gen7_append_oa_reports(stream, buf, count, offset); in gen7_oa_read()
1148 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) in i915_oa_wait_unlocked() argument
1151 if (!stream->periodic) in i915_oa_wait_unlocked()
1154 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1155 oa_buffer_check_unlocked(stream)); in i915_oa_wait_unlocked()
1168 static void i915_oa_poll_wait(struct i915_perf_stream *stream, in i915_oa_poll_wait() argument
1172 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1187 static int i915_oa_read(struct i915_perf_stream *stream, in i915_oa_read() argument
1192 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1195 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) in oa_pin_context() argument
1198 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1204 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1232 stream->pinned_ctx = ce; in oa_pin_context()
1233 return stream->pinned_ctx; in oa_pin_context()
1246 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) in oa_get_render_ctx_id() argument
1250 ce = oa_pin_context(stream); in oa_get_render_ctx_id()
1260 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1261 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1269 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1271 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1283 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1289 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1296 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1303 stream->specific_ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); in oa_get_render_ctx_id()
1311 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1313 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1315 stream->specific_ctx_id, in oa_get_render_ctx_id()
1316 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1328 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) in oa_put_render_ctx_id() argument
1332 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1338 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1339 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1343 free_oa_buffer(struct i915_perf_stream *stream) in free_oa_buffer() argument
1345 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1348 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1352 free_oa_configs(struct i915_perf_stream *stream) in free_oa_configs() argument
1356 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1357 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1362 free_noa_wait(struct i915_perf_stream *stream) in free_noa_wait() argument
1364 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1367 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) in i915_oa_stream_destroy() argument
1369 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1371 BUG_ON(stream != perf->exclusive_stream); in i915_oa_stream_destroy()
1380 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1382 free_oa_buffer(stream); in i915_oa_stream_destroy()
1384 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1385 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1387 if (stream->ctx) in i915_oa_stream_destroy()
1388 oa_put_render_ctx_id(stream); in i915_oa_stream_destroy()
1390 free_oa_configs(stream); in i915_oa_stream_destroy()
1391 free_noa_wait(stream); in i915_oa_stream_destroy()
1399 static void gen7_init_oa_buffer(struct i915_perf_stream *stream) in gen7_init_oa_buffer() argument
1401 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1402 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1405 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1412 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1420 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1421 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1423 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1429 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1442 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1445 static void gen8_init_oa_buffer(struct i915_perf_stream *stream) in gen8_init_oa_buffer() argument
1447 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1448 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1451 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1455 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1472 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1473 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1480 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1482 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1496 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1499 static void gen12_init_oa_buffer(struct i915_perf_stream *stream) in gen12_init_oa_buffer() argument
1501 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1502 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1505 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1510 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1526 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen12_init_oa_buffer()
1527 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1534 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1536 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1550 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1551 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1554 static int alloc_oa_buffer(struct i915_perf_stream *stream) in alloc_oa_buffer() argument
1556 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1561 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1567 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1581 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1583 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1585 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1586 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1598 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1599 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1604 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, in save_restore_register() argument
1613 if (INTEL_GEN(stream->perf->i915) >= 8) in save_restore_register()
1619 *cs++ = intel_gt_scratch_offset(stream->engine->gt, in save_restore_register()
1627 static int alloc_noa_wait(struct i915_perf_stream *stream) in alloc_noa_wait() argument
1629 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1633 i915_cs_timestamp_ns_to_ticks(i915, atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1634 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1674 stream, cs, true /* save */, CS_GPR(i), in alloc_noa_wait()
1677 stream, cs, true /* save */, MI_PREDICATE_RESULT_1, in alloc_noa_wait()
1781 stream, cs, false /* restore */, CS_GPR(i), in alloc_noa_wait()
1784 stream, cs, false /* restore */, MI_PREDICATE_RESULT_1, in alloc_noa_wait()
1795 stream->noa_wait = vma; in alloc_noa_wait()
1839 alloc_oa_config_buffer(struct i915_perf_stream *stream, in alloc_oa_config_buffer() argument
1858 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
1881 *cs++ = (INTEL_GEN(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
1884 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
1891 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
1899 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
1911 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config) in get_oa_vma() argument
1919 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
1927 oa_bo = alloc_oa_config_buffer(stream, oa_config); in get_oa_vma()
1936 emit_oa_config(struct i915_perf_stream *stream, in emit_oa_config() argument
1946 vma = get_oa_vma(stream, oa_config); in emit_oa_config()
2008 static struct intel_context *oa_context(struct i915_perf_stream *stream) in oa_context() argument
2010 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2014 hsw_enable_metric_set(struct i915_perf_stream *stream, in hsw_enable_metric_set() argument
2017 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2034 return emit_oa_config(stream, in hsw_enable_metric_set()
2035 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2039 static void hsw_disable_metric_set(struct i915_perf_stream *stream) in hsw_disable_metric_set() argument
2041 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2081 const struct i915_perf_stream *stream) in gen8_update_reg_state_unlocked() argument
2083 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2084 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2099 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2100 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2105 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2241 static int gen12_configure_oar_context(struct i915_perf_stream *stream, in gen12_configure_oar_context() argument
2245 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2246 u32 format = stream->oa_buffer.format; in gen12_configure_oar_context()
2250 stream->perf->ctx_oactxctrl_offset + 1, in gen12_configure_oar_context()
2315 oa_configure_all_contexts(struct i915_perf_stream *stream, in oa_configure_all_contexts() argument
2320 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2325 lockdep_assert_held(&stream->perf->lock); in oa_configure_all_contexts()
2384 gen12_configure_all_contexts(struct i915_perf_stream *stream, in gen12_configure_all_contexts() argument
2395 return oa_configure_all_contexts(stream, in gen12_configure_all_contexts()
2401 lrc_configure_all_contexts(struct i915_perf_stream *stream, in lrc_configure_all_contexts() argument
2406 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2415 stream->perf->ctx_oactxctrl_offset + 1, in lrc_configure_all_contexts()
2429 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2430 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2436 return oa_configure_all_contexts(stream, in lrc_configure_all_contexts()
2442 gen8_enable_metric_set(struct i915_perf_stream *stream, in gen8_enable_metric_set() argument
2445 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2446 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2472 if (IS_GEN_RANGE(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2483 ret = lrc_configure_all_contexts(stream, oa_config, active); in gen8_enable_metric_set()
2487 return emit_oa_config(stream, in gen8_enable_metric_set()
2488 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2492 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) in oag_report_ctx_switches() argument
2495 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2500 gen12_enable_metric_set(struct i915_perf_stream *stream, in gen12_enable_metric_set() argument
2503 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2504 struct i915_oa_config *oa_config = stream->oa_config; in gen12_enable_metric_set()
2505 bool periodic = stream->periodic; in gen12_enable_metric_set()
2506 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2517 oag_report_ctx_switches(stream)); in gen12_enable_metric_set()
2530 ret = gen12_configure_all_contexts(stream, oa_config, active); in gen12_enable_metric_set()
2539 if (stream->ctx) { in gen12_enable_metric_set()
2540 ret = gen12_configure_oar_context(stream, active); in gen12_enable_metric_set()
2545 return emit_oa_config(stream, in gen12_enable_metric_set()
2546 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2550 static void gen8_disable_metric_set(struct i915_perf_stream *stream) in gen8_disable_metric_set() argument
2552 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2555 lrc_configure_all_contexts(stream, NULL, NULL); in gen8_disable_metric_set()
2560 static void gen10_disable_metric_set(struct i915_perf_stream *stream) in gen10_disable_metric_set() argument
2562 struct intel_uncore *uncore = stream->uncore; in gen10_disable_metric_set()
2565 lrc_configure_all_contexts(stream, NULL, NULL); in gen10_disable_metric_set()
2571 static void gen12_disable_metric_set(struct i915_perf_stream *stream) in gen12_disable_metric_set() argument
2573 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2576 gen12_configure_all_contexts(stream, NULL, NULL); in gen12_disable_metric_set()
2579 if (stream->ctx) in gen12_disable_metric_set()
2580 gen12_configure_oar_context(stream, NULL); in gen12_disable_metric_set()
2586 static void gen7_oa_enable(struct i915_perf_stream *stream) in gen7_oa_enable() argument
2588 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
2589 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2590 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2591 bool periodic = stream->periodic; in gen7_oa_enable()
2592 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2593 u32 report_format = stream->oa_buffer.format; in gen7_oa_enable()
2604 gen7_init_oa_buffer(stream); in gen7_oa_enable()
2616 static void gen8_oa_enable(struct i915_perf_stream *stream) in gen8_oa_enable() argument
2618 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
2619 u32 report_format = stream->oa_buffer.format; in gen8_oa_enable()
2630 gen8_init_oa_buffer(stream); in gen8_oa_enable()
2642 static void gen12_oa_enable(struct i915_perf_stream *stream) in gen12_oa_enable() argument
2644 struct intel_uncore *uncore = stream->uncore; in gen12_oa_enable()
2645 u32 report_format = stream->oa_buffer.format; in gen12_oa_enable()
2651 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
2654 gen12_init_oa_buffer(stream); in gen12_oa_enable()
2670 static void i915_oa_stream_enable(struct i915_perf_stream *stream) in i915_oa_stream_enable() argument
2672 stream->pollin = false; in i915_oa_stream_enable()
2674 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
2676 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
2677 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
2678 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
2682 static void gen7_oa_disable(struct i915_perf_stream *stream) in gen7_oa_disable() argument
2684 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
2690 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
2694 static void gen8_oa_disable(struct i915_perf_stream *stream) in gen8_oa_disable() argument
2696 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
2702 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
2706 static void gen12_oa_disable(struct i915_perf_stream *stream) in gen12_oa_disable() argument
2708 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
2715 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2723 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2735 static void i915_oa_stream_disable(struct i915_perf_stream *stream) in i915_oa_stream_disable() argument
2737 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
2739 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
2740 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
2752 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) in i915_perf_stream_enable_sync() argument
2761 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
2819 static int i915_oa_stream_init(struct i915_perf_stream *stream, in i915_oa_stream_init() argument
2823 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
2824 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
2844 (INTEL_GEN(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
2869 stream->engine = props->engine; in i915_oa_stream_init()
2870 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
2872 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
2876 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
2877 stream->sample_size += format_size; in i915_oa_stream_init()
2879 stream->oa_buffer.format_size = format_size; in i915_oa_stream_init()
2880 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0)) in i915_oa_stream_init()
2883 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
2885 stream->oa_buffer.format = in i915_oa_stream_init()
2888 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
2889 if (stream->periodic) in i915_oa_stream_init()
2890 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2892 if (stream->ctx) { in i915_oa_stream_init()
2893 ret = oa_get_render_ctx_id(stream); in i915_oa_stream_init()
2900 ret = alloc_noa_wait(stream); in i915_oa_stream_init()
2906 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
2907 if (!stream->oa_config) { in i915_oa_stream_init()
2925 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
2926 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2928 ret = alloc_oa_buffer(stream); in i915_oa_stream_init()
2932 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
2935 WRITE_ONCE(perf->exclusive_stream, stream); in i915_oa_stream_init()
2937 ret = i915_perf_stream_enable_sync(stream); in i915_oa_stream_init()
2944 stream->oa_config->uuid); in i915_oa_stream_init()
2946 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
2948 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
2949 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
2950 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
2956 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
2958 free_oa_buffer(stream); in i915_oa_stream_init()
2961 free_oa_configs(stream); in i915_oa_stream_init()
2963 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2964 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
2967 free_noa_wait(stream); in i915_oa_stream_init()
2970 if (stream->ctx) in i915_oa_stream_init()
2971 oa_put_render_ctx_id(stream); in i915_oa_stream_init()
2979 struct i915_perf_stream *stream; in i915_oa_init_reg_state() local
2985 stream = READ_ONCE(engine->i915->perf.exclusive_stream); in i915_oa_init_reg_state()
2986 if (stream && INTEL_GEN(stream->perf->i915) < 12) in i915_oa_init_reg_state()
2987 gen8_update_reg_state_unlocked(ce, stream); in i915_oa_init_reg_state()
3013 struct i915_perf_stream *stream = file->private_data; in i915_perf_read() local
3014 struct i915_perf *perf = stream->perf; in i915_perf_read()
3022 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3034 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3039 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3044 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3060 stream->pollin = false; in i915_perf_read()
3068 struct i915_perf_stream *stream = in oa_poll_check_timer_cb() local
3069 container_of(hrtimer, typeof(*stream), poll_check_timer); in oa_poll_check_timer_cb()
3071 if (oa_buffer_check_unlocked(stream)) { in oa_poll_check_timer_cb()
3072 stream->pollin = true; in oa_poll_check_timer_cb()
3073 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3077 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3097 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream, in i915_perf_poll_locked() argument
3103 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3111 if (stream->pollin) in i915_perf_poll_locked()
3132 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll() local
3133 struct i915_perf *perf = stream->perf; in i915_perf_poll()
3137 ret = i915_perf_poll_locked(stream, file, wait); in i915_perf_poll()
3153 static void i915_perf_enable_locked(struct i915_perf_stream *stream) in i915_perf_enable_locked() argument
3155 if (stream->enabled) in i915_perf_enable_locked()
3159 stream->enabled = true; in i915_perf_enable_locked()
3161 if (stream->ops->enable) in i915_perf_enable_locked()
3162 stream->ops->enable(stream); in i915_perf_enable_locked()
3164 if (stream->hold_preemption) in i915_perf_enable_locked()
3165 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3182 static void i915_perf_disable_locked(struct i915_perf_stream *stream) in i915_perf_disable_locked() argument
3184 if (!stream->enabled) in i915_perf_disable_locked()
3188 stream->enabled = false; in i915_perf_disable_locked()
3190 if (stream->hold_preemption) in i915_perf_disable_locked()
3191 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3193 if (stream->ops->disable) in i915_perf_disable_locked()
3194 stream->ops->disable(stream); in i915_perf_disable_locked()
3197 static long i915_perf_config_locked(struct i915_perf_stream *stream, in i915_perf_config_locked() argument
3201 long ret = stream->oa_config->id; in i915_perf_config_locked()
3203 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3207 if (config != stream->oa_config) { in i915_perf_config_locked()
3219 err = emit_oa_config(stream, config, oa_context(stream), NULL); in i915_perf_config_locked()
3221 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3243 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, in i915_perf_ioctl_locked() argument
3249 i915_perf_enable_locked(stream); in i915_perf_ioctl_locked()
3252 i915_perf_disable_locked(stream); in i915_perf_ioctl_locked()
3255 return i915_perf_config_locked(stream, arg); in i915_perf_ioctl_locked()
3276 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl() local
3277 struct i915_perf *perf = stream->perf; in i915_perf_ioctl()
3281 ret = i915_perf_ioctl_locked(stream, cmd, arg); in i915_perf_ioctl()
3297 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) in i915_perf_destroy_locked() argument
3299 if (stream->enabled) in i915_perf_destroy_locked()
3300 i915_perf_disable_locked(stream); in i915_perf_destroy_locked()
3302 if (stream->ops->destroy) in i915_perf_destroy_locked()
3303 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3305 if (stream->ctx) in i915_perf_destroy_locked()
3306 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3308 kfree(stream); in i915_perf_destroy_locked()
3324 struct i915_perf_stream *stream = file->private_data; in i915_perf_release() local
3325 struct i915_perf *perf = stream->perf; in i915_perf_release()
3328 i915_perf_destroy_locked(stream); in i915_perf_release()
3383 struct i915_perf_stream *stream = NULL; in i915_perf_open_ioctl_locked() local
3456 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in i915_perf_open_ioctl_locked()
3457 if (!stream) { in i915_perf_open_ioctl_locked()
3462 stream->perf = perf; in i915_perf_open_ioctl_locked()
3463 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3464 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3466 ret = i915_oa_stream_init(stream, param, props); in i915_perf_open_ioctl_locked()
3474 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3484 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); in i915_perf_open_ioctl_locked()
3491 i915_perf_enable_locked(stream); in i915_perf_open_ioctl_locked()
3501 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3502 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3504 kfree(stream); in i915_perf_open_ioctl_locked()