60 static void si_query_sw_destroy(struct si_context *sctx, struct si_query *squery)  in si_query_sw_destroy()  argument
64    sctx->b.screen->fence_reference(sctx->b.screen, &query->fence, NULL);  in si_query_sw_destroy()
116 static int64_t si_finish_dma_get_cpu_time(struct si_context *sctx)  in si_finish_dma_get_cpu_time()  argument
120    si_flush_dma_cs(sctx, 0, &fence);  in si_finish_dma_get_cpu_time()
122       sctx->ws->fence_wait(sctx->ws, fence, PIPE_TIMEOUT_INFINITE);  in si_finish_dma_get_cpu_time()
123       sctx->ws->fence_reference(&fence, NULL);  in si_finish_dma_get_cpu_time()
129 static bool si_query_sw_begin(struct si_context *sctx, struct si_query *squery)  in si_query_sw_begin()  argument
139       query->begin_result = si_finish_dma_get_cpu_time(sctx);  in si_query_sw_begin()
142       query->begin_result = sctx->num_draw_calls;  in si_query_sw_begin()
145       query->begin_result = sctx->num_decompress_calls;  in si_query_sw_begin()
148       query->begin_result = sctx->num_mrt_draw_calls;  in si_query_sw_begin()
151       query->begin_result = sctx->num_prim_restart_calls;  in si_query_sw_begin()
154       query->begin_result = sctx->num_spill_draw_calls;  in si_query_sw_begin()
157       query->begin_result = sctx->num_compute_calls;  in si_query_sw_begin()
160       query->begin_result = sctx->num_spill_compute_calls;  in si_query_sw_begin()
163       query->begin_result = sctx->num_dma_calls;  in si_query_sw_begin()
166       query->begin_result = sctx->num_cp_dma_calls;  in si_query_sw_begin()
169       query->begin_result = sctx->num_vs_flushes;  in si_query_sw_begin()
172       query->begin_result = sctx->num_ps_flushes;  in si_query_sw_begin()
175       query->begin_result = sctx->num_cs_flushes;  in si_query_sw_begin()
178       query->begin_result = sctx->num_cb_cache_flushes;  in si_query_sw_begin()
181       query->begin_result = sctx->num_db_cache_flushes;  in si_query_sw_begin()
184       query->begin_result = sctx->num_L2_invalidates;  in si_query_sw_begin()
187       query->begin_result = sctx->num_L2_writebacks;  in si_query_sw_begin()
190       query->begin_result = sctx->num_resident_handles;  in si_query_sw_begin()
193       query->begin_result = sctx->tc ? sctx->tc->num_offloaded_slots : 0;  in si_query_sw_begin()
196       query->begin_result = sctx->tc ? sctx->tc->num_direct_slots : 0;  in si_query_sw_begin()
199       query->begin_result = sctx->tc ? sctx->tc->num_syncs : 0;  in si_query_sw_begin()
223       query->begin_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_begin()
228       query->begin_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_begin()
229       query->begin_time = sctx->ws->query_value(sctx->ws, RADEON_NUM_GFX_IBS);  in si_query_sw_begin()
233       query->begin_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_begin()
237       query->begin_result = sctx->tc ? util_queue_get_thread_time_nano(&sctx->tc->queue, 0) : 0;  in si_query_sw_begin()
261       query->begin_result = si_begin_counter(sctx->screen, query->b.type);  in si_query_sw_begin()
264       query->begin_result = p_atomic_read(&sctx->screen->num_compilations);  in si_query_sw_begin()
267       query->begin_result = p_atomic_read(&sctx->screen->num_shaders_created);  in si_query_sw_begin()
270       query->begin_result = sctx->screen->live_shader_cache.hits;  in si_query_sw_begin()
273       query->begin_result = sctx->screen->live_shader_cache.misses;  in si_query_sw_begin()
276       query->begin_result = sctx->screen->num_memory_shader_cache_hits;  in si_query_sw_begin()
279       query->begin_result = sctx->screen->num_memory_shader_cache_misses;  in si_query_sw_begin()
282       query->begin_result = sctx->screen->num_disk_shader_cache_hits;  in si_query_sw_begin()
285       query->begin_result = sctx->screen->num_disk_shader_cache_misses;  in si_query_sw_begin()
288       query->begin_result = sctx->compute_num_verts_accepted;  in si_query_sw_begin()
291       query->begin_result = sctx->compute_num_verts_rejected;  in si_query_sw_begin()
294       query->begin_result = sctx->compute_num_verts_ineligible;  in si_query_sw_begin()
309 static bool si_query_sw_end(struct si_context *sctx, struct si_query *squery)  in si_query_sw_end()  argument
318       sctx->b.flush(&sctx->b, &query->fence, PIPE_FLUSH_DEFERRED);  in si_query_sw_end()
321       query->end_result = si_finish_dma_get_cpu_time(sctx);  in si_query_sw_end()
324       query->end_result = sctx->num_draw_calls;  in si_query_sw_end()
327       query->end_result = sctx->num_decompress_calls;  in si_query_sw_end()
330       query->end_result = sctx->num_mrt_draw_calls;  in si_query_sw_end()
333       query->end_result = sctx->num_prim_restart_calls;  in si_query_sw_end()
336       query->end_result = sctx->num_spill_draw_calls;  in si_query_sw_end()
339       query->end_result = sctx->num_compute_calls;  in si_query_sw_end()
342       query->end_result = sctx->num_spill_compute_calls;  in si_query_sw_end()
345       query->end_result = sctx->num_dma_calls;  in si_query_sw_end()
348       query->end_result = sctx->num_cp_dma_calls;  in si_query_sw_end()
351       query->end_result = sctx->num_vs_flushes;  in si_query_sw_end()
354       query->end_result = sctx->num_ps_flushes;  in si_query_sw_end()
357       query->end_result = sctx->num_cs_flushes;  in si_query_sw_end()
360       query->end_result = sctx->num_cb_cache_flushes;  in si_query_sw_end()
363       query->end_result = sctx->num_db_cache_flushes;  in si_query_sw_end()
366       query->end_result = sctx->num_L2_invalidates;  in si_query_sw_end()
369       query->end_result = sctx->num_L2_writebacks;  in si_query_sw_end()
372       query->end_result = sctx->num_resident_handles;  in si_query_sw_end()
375       query->end_result = sctx->tc ? sctx->tc->num_offloaded_slots : 0;  in si_query_sw_end()
378       query->end_result = sctx->tc ? sctx->tc->num_direct_slots : 0;  in si_query_sw_end()
381       query->end_result = sctx->tc ? sctx->tc->num_syncs : 0;  in si_query_sw_end()
402       query->end_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_end()
407       query->end_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_end()
408       query->end_time = sctx->ws->query_value(sctx->ws, RADEON_NUM_GFX_IBS);  in si_query_sw_end()
412       query->end_result = sctx->ws->query_value(sctx->ws, ws_id);  in si_query_sw_end()
416       query->end_result = sctx->tc ? util_queue_get_thread_time_nano(&sctx->tc->queue, 0) : 0;  in si_query_sw_end()
440       query->end_result = si_end_counter(sctx->screen, query->b.type, query->begin_result);  in si_query_sw_end()
444       query->end_result = p_atomic_read(&sctx->screen->num_compilations);  in si_query_sw_end()
447       query->end_result = p_atomic_read(&sctx->screen->num_shaders_created);  in si_query_sw_end()
450       query->end_result = sctx->last_tex_ps_draw_ratio;  in si_query_sw_end()
453       query->end_result = sctx->screen->live_shader_cache.hits;  in si_query_sw_end()
456       query->end_result = sctx->screen->live_shader_cache.misses;  in si_query_sw_end()
459       query->end_result = sctx->screen->num_memory_shader_cache_hits;  in si_query_sw_end()
462       query->end_result = sctx->screen->num_memory_shader_cache_misses;  in si_query_sw_end()
465       query->end_result = sctx->screen->num_disk_shader_cache_hits;  in si_query_sw_end()
468       query->end_result = sctx->screen->num_disk_shader_cache_misses;  in si_query_sw_end()
471       query->end_result = sctx->compute_num_verts_accepted;  in si_query_sw_end()
474       query->end_result = sctx->compute_num_verts_rejected;  in si_query_sw_end()
477       query->end_result = sctx->compute_num_verts_ineligible;  in si_query_sw_end()
492 static bool si_query_sw_get_result(struct si_context *sctx, struct si_query *squery, bool wait,  in si_query_sw_get_result()  argument
500       result->timestamp_disjoint.frequency = (uint64_t)sctx->screen->info.clock_crystal_freq * 1000;  in si_query_sw_get_result()
504       struct pipe_screen *screen = sctx->b.screen;  in si_query_sw_get_result()
505       struct pipe_context *ctx = squery->b.flushed ? NULL : &sctx->b;  in si_query_sw_get_result()
529       result->u32 = sctx->screen->info.num_good_compute_units;  in si_query_sw_get_result()
532       result->u32 = sctx->screen->info.num_render_backends;  in si_query_sw_get_result()
538       result->u32 = sctx->screen->info.max_se;  in si_query_sw_get_result()
593 void si_query_buffer_reset(struct si_context *sctx, struct si_query_buffer *buffer)  in si_query_buffer_reset()  argument
610    if (si_rings_is_buffer_referenced(sctx, buffer->buf->buf, RADEON_USAGE_READWRITE) ||  in si_query_buffer_reset()
611        !sctx->ws->buffer_wait(buffer->buf->buf, 0, RADEON_USAGE_READWRITE)) {  in si_query_buffer_reset()
618 bool si_query_buffer_alloc(struct si_context *sctx, struct si_query_buffer *buffer,  in si_query_buffer_alloc()  argument
637       struct si_screen *screen = sctx->screen;  in si_query_buffer_alloc()
646       if (unlikely(!prepare_buffer(sctx, buffer))) {  in si_query_buffer_alloc()
655 void si_query_hw_destroy(struct si_context *sctx, struct si_query *squery)  in si_query_hw_destroy()  argument
659    si_query_buffer_destroy(sctx->screen, &query->buffer);  in si_query_hw_destroy()
664 static bool si_query_hw_prepare_buffer(struct si_context *sctx, struct si_query_buffer *qbuf)  in si_query_hw_prepare_buffer()  argument
668    struct si_screen *screen = sctx->screen;  in si_query_hw_prepare_buffer()
702 static void si_query_hw_get_result_resource(struct si_context *sctx, struct si_query *squery,
707 static void si_query_hw_do_emit_start(struct si_context *sctx, struct si_query_hw *query,
709 static void si_query_hw_do_emit_stop(struct si_context *sctx, struct si_query_hw *query,
784 static void si_update_occlusion_query_state(struct si_context *sctx, unsigned type, int diff)  in si_update_occlusion_query_state()  argument
788       bool old_enable = sctx->num_occlusion_queries != 0;  in si_update_occlusion_query_state()
789       bool old_perfect_enable = sctx->num_perfect_occlusion_queries != 0;  in si_update_occlusion_query_state()
792       sctx->num_occlusion_queries += diff;  in si_update_occlusion_query_state()
793       assert(sctx->num_occlusion_queries >= 0);  in si_update_occlusion_query_state()
796          sctx->num_perfect_occlusion_queries += diff;  in si_update_occlusion_query_state()
797          assert(sctx->num_perfect_occlusion_queries >= 0);  in si_update_occlusion_query_state()
800       enable = sctx->num_occlusion_queries != 0;  in si_update_occlusion_query_state()
801       perfect_enable = sctx->num_perfect_occlusion_queries != 0;  in si_update_occlusion_query_state()
804          si_set_occlusion_query_state(sctx, old_perfect_enable);  in si_update_occlusion_query_state()
832 static void si_query_hw_do_emit_start(struct si_context *sctx, struct si_query_hw *query,  in si_query_hw_do_emit_start()  argument
835    struct radeon_cmdbuf *cs = sctx->gfx_cs;  in si_query_hw_do_emit_start()
839       si_dma_emit_timestamp(sctx, buffer, va - buffer->gpu_address);  in si_query_hw_do_emit_start()
860       si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,  in si_query_hw_do_emit_start()
872    radeon_add_to_buffer_list(sctx, sctx->gfx_cs, query->buffer.buf, RADEON_USAGE_WRITE,  in si_query_hw_do_emit_start()
876 static void si_query_hw_emit_start(struct si_context *sctx, struct si_query_hw *query)  in si_query_hw_emit_start()  argument
880    if (!si_query_buffer_alloc(sctx, &query->buffer, query->ops->prepare_buffer, query->result_size))  in si_query_hw_emit_start()
883    si_update_occlusion_query_state(sctx, query->b.type, 1);  in si_query_hw_emit_start()
884    si_update_prims_generated_query_state(sctx, query->b.type, 1);  in si_query_hw_emit_start()
887       sctx->num_pipeline_stat_queries++;  in si_query_hw_emit_start()
890       si_need_gfx_cs_space(sctx, 0);  in si_query_hw_emit_start()
893    query->ops->emit_start(sctx, query, query->buffer.buf, va);  in si_query_hw_emit_start()
896 static void si_query_hw_do_emit_stop(struct si_context *sctx, struct si_query_hw *query,  in si_query_hw_do_emit_stop()  argument
899    struct radeon_cmdbuf *cs = sctx->gfx_cs;  in si_query_hw_do_emit_stop()
904       si_dma_emit_timestamp(sctx, buffer, va + 32 - buffer->gpu_address);  in si_query_hw_do_emit_stop()
915       fence_va = va + sctx->screen->info.num_render_backends * 16 - 8;  in si_query_hw_do_emit_stop()
933       si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,  in si_query_hw_do_emit_stop()
952    radeon_add_to_buffer_list(sctx, sctx->gfx_cs, query->buffer.buf, RADEON_USAGE_WRITE,  in si_query_hw_do_emit_stop()
956       si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,  in si_query_hw_do_emit_stop()
962 static void si_query_hw_emit_stop(struct si_context *sctx, struct si_query_hw *query)  in si_query_hw_emit_stop()  argument
968       si_need_gfx_cs_space(sctx, 0);  in si_query_hw_emit_stop()
969       if (!si_query_buffer_alloc(sctx, &query->buffer, query->ops->prepare_buffer,  in si_query_hw_emit_stop()
980    query->ops->emit_stop(sctx, query, query->buffer.buf, va);  in si_query_hw_emit_stop()
984    si_update_occlusion_query_state(sctx, query->b.type, -1);  in si_query_hw_emit_stop()
985    si_update_prims_generated_query_state(sctx, query->b.type, -1);  in si_query_hw_emit_stop()
988       sctx->num_pipeline_stat_queries--;  in si_query_hw_emit_stop()
1115    struct si_context *sctx = (struct si_context *)ctx;  in si_destroy_query()  local
1118    squery->ops->destroy(sctx, squery);  in si_destroy_query()
1123    struct si_context *sctx = (struct si_context *)ctx;  in si_begin_query()  local
1126    return squery->ops->begin(sctx, squery);  in si_begin_query()
1129 bool si_query_hw_begin(struct si_context *sctx, struct si_query *squery)  in si_query_hw_begin()  argument
1139       si_query_buffer_reset(sctx, &query->buffer);  in si_query_hw_begin()
1143    si_query_hw_emit_start(sctx, query);  in si_query_hw_begin()
1147    list_addtail(&query->b.active_list, &sctx->active_queries);  in si_query_hw_begin()
1148    sctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend;  in si_query_hw_begin()
1154    struct si_context *sctx = (struct si_context *)ctx;  in si_end_query()  local
1157    return squery->ops->end(sctx, squery);  in si_end_query()
1160 bool si_query_hw_end(struct si_context *sctx, struct si_query *squery)  in si_query_hw_end()  argument
1165       si_query_buffer_reset(sctx, &query->buffer);  in si_query_hw_end()
1167    si_query_hw_emit_stop(sctx, query);  in si_query_hw_end()
1171       sctx->num_cs_dw_queries_suspend -= query->b.num_cs_dw_suspend;  in si_query_hw_end()
1180 static void si_get_hw_query_params(struct si_context *sctx, struct si_query_hw *squery, int index,  in si_get_hw_query_params()  argument
1183    unsigned max_rbs = sctx->screen->info.num_render_backends;  in si_get_hw_query_params()
1356 void si_query_hw_suspend(struct si_context *sctx, struct si_query *query)  in si_query_hw_suspend()  argument
1358    si_query_hw_emit_stop(sctx, (struct si_query_hw *)query);  in si_query_hw_suspend()
1361 void si_query_hw_resume(struct si_context *sctx, struct si_query *query)  in si_query_hw_resume()  argument
1363    si_query_hw_emit_start(sctx, (struct si_query_hw *)query);  in si_query_hw_resume()
1380    struct si_context *sctx = (struct si_context *)ctx;  in si_get_query_result()  local
1383    return squery->ops->get_result(sctx, squery, wait, result);  in si_get_query_result()
1390    struct si_context *sctx = (struct si_context *)ctx;  in si_get_query_result_resource()  local
1393    squery->ops->get_result_resource(sctx, squery, wait, result_type, index, resource, offset);  in si_get_query_result_resource()
1401 bool si_query_hw_get_result(struct si_context *sctx, struct si_query *squery, bool wait,  in si_query_hw_get_result()  argument
1404    struct si_screen *sscreen = sctx->screen;  in si_query_hw_get_result()
1416          map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);  in si_query_hw_get_result()
1418          map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);  in si_query_hw_get_result()
1437 static void si_query_hw_get_result_resource(struct si_context *sctx, struct si_query *squery,  in si_query_hw_get_result_resource()  argument
1462    if (!sctx->query_result_shader) {  in si_query_hw_get_result_resource()
1463       sctx->query_result_shader = si_create_query_result_cs(sctx);  in si_query_hw_get_result_resource()
1464       if (!sctx->query_result_shader)  in si_query_hw_get_result_resource()
1469       u_suballocator_alloc(sctx->allocator_zeroed_memory, 16, 16, &tmp_buffer_offset, &tmp_buffer);  in si_query_hw_get_result_resource()
1474    si_save_qbo_state(sctx, &saved_state);  in si_query_hw_get_result_resource()
1476    si_get_hw_query_params(sctx, query, index >= 0 ? index : 0, &params);  in si_query_hw_get_result_resource()
1492    sctx->b.bind_compute_state(&sctx->b, sctx->query_result_shader);  in si_query_hw_get_result_resource()
1525    sctx->flags |= sctx->screen->barrier_flags.cp_to_L2;  in si_query_hw_get_result_resource()
1544       sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, &constant_buffer);  in si_query_hw_get_result_resource()
1558       sctx->b.set_shader_buffers(&sctx->b, PIPE_SHADER_COMPUTE, 0, 3, ssbo, 1 << 2);  in si_query_hw_get_result_resource()
1570          si_cp_wait_mem(sctx, sctx->gfx_cs, va, 0x80000000, 0x80000000, WAIT_REG_MEM_EQUAL);  in si_query_hw_get_result_resource()
1573       sctx->b.launch_grid(&sctx->b, &grid);  in si_query_hw_get_result_resource()
1574       sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;  in si_query_hw_get_result_resource()
1577    si_restore_qbo_state(sctx, &saved_state);  in si_query_hw_get_result_resource()
1584    struct si_context *sctx = (struct si_context *)ctx;  in si_render_condition()  local
1586    struct si_atom *atom = &sctx->atoms.s.render_cond;  in si_render_condition()
1595       if (((sctx->chip_class == GFX8 && sctx->screen->info.pfp_fw_feature < 49) ||  in si_render_condition()
1596            (sctx->chip_class == GFX9 && sctx->screen->info.pfp_fw_feature < 38)) &&  in si_render_condition()
1605          bool old_force_off = sctx->render_cond_force_off;  in si_render_condition()
1606          sctx->render_cond_force_off = true;  in si_render_condition()
1608          u_suballocator_alloc(sctx->allocator_zeroed_memory, 8, 8, &squery->workaround_offset,  in si_render_condition()
1614          sctx->render_cond = NULL;  in si_render_condition()
1621          sctx->flags |= sctx->screen->barrier_flags.L2_to_cp | SI_CONTEXT_FLUSH_FOR_RENDER_COND;  in si_render_condition()
1623          sctx->render_cond_force_off = old_force_off;  in si_render_condition()
1627    sctx->render_cond = query;  in si_render_condition()
1628    sctx->render_cond_invert = condition;  in si_render_condition()
1629    sctx->render_cond_mode = mode;  in si_render_condition()
1631    si_set_atom_dirty(sctx, atom, query != NULL);  in si_render_condition()
1634 void si_suspend_queries(struct si_context *sctx)  in si_suspend_queries()  argument
1638    LIST_FOR_EACH_ENTRY (query, &sctx->active_queries, active_list)  in si_suspend_queries()
1639       query->ops->suspend(sctx, query);  in si_suspend_queries()
1642 void si_resume_queries(struct si_context *sctx)  in si_resume_queries()  argument
1647    si_need_gfx_cs_space(sctx, 0);  in si_resume_queries()
1649    LIST_FOR_EACH_ENTRY (query, &sctx->active_queries, active_list)  in si_resume_queries()
1650       query->ops->resume(sctx, query);  in si_resume_queries()
1858 void si_init_query_functions(struct si_context *sctx)  in si_init_query_functions()  argument
1860    sctx->b.create_query = si_create_query;  in si_init_query_functions()
1861    sctx->b.create_batch_query = si_create_batch_query;  in si_init_query_functions()
1862    sctx->b.destroy_query = si_destroy_query;  in si_init_query_functions()
1863    sctx->b.begin_query = si_begin_query;  in si_init_query_functions()
1864    sctx->b.end_query = si_end_query;  in si_init_query_functions()
1865    sctx->b.get_query_result = si_get_query_result;  in si_init_query_functions()
1866    sctx->b.get_query_result_resource = si_get_query_result_resource;  in si_init_query_functions()
1868    if (sctx->has_graphics) {  in si_init_query_functions()
1869       sctx->atoms.s.render_cond.emit = si_emit_query_predication;  in si_init_query_functions()
1870       sctx->b.render_condition = si_render_condition;  in si_init_query_functions()
1873    list_inithead(&sctx->active_queries);  in si_init_query_functions()