Lines Matching full:device
96 VK_FROM_HANDLE(radv_device, device, _device); in radv_GetMemoryHostPointerPropertiesEXT()
97 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_GetMemoryHostPointerPropertiesEXT()
117 radv_device_init_border_color(struct radv_device *device) in radv_device_init_border_color() argument
121 result = radv_bo_create(device, NULL, RADV_BORDER_COLOR_BUFFER_SIZE, 4096, RADEON_DOMAIN_VRAM, in radv_device_init_border_color()
123 RADV_BO_PRIORITY_SHADER, 0, true, &device->border_color_data.bo); in radv_device_init_border_color()
126 return vk_error(device, result); in radv_device_init_border_color()
128 radv_rmv_log_border_color_palette_create(device, device->border_color_data.bo); in radv_device_init_border_color()
130 result = device->ws->buffer_make_resident(device->ws, device->border_color_data.bo, true); in radv_device_init_border_color()
132 return vk_error(device, result); in radv_device_init_border_color()
134 …device->border_color_data.colors_gpu_ptr = radv_buffer_map(device->ws, device->border_color_data.b… in radv_device_init_border_color()
135 if (!device->border_color_data.colors_gpu_ptr) in radv_device_init_border_color()
136 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); in radv_device_init_border_color()
137 mtx_init(&device->border_color_data.mutex, mtx_plain); in radv_device_init_border_color()
143 radv_device_finish_border_color(struct radv_device *device) in radv_device_finish_border_color() argument
145 if (device->border_color_data.bo) { in radv_device_finish_border_color()
146 radv_rmv_log_border_color_palette_destroy(device, device->border_color_data.bo); in radv_device_finish_border_color()
147 device->ws->buffer_make_resident(device->ws, device->border_color_data.bo, false); in radv_device_finish_border_color()
148 radv_bo_destroy(device, NULL, device->border_color_data.bo); in radv_device_finish_border_color()
150 mtx_destroy(&device->border_color_data.mutex); in radv_device_finish_border_color()
155 _radv_create_vs_prolog(struct radv_device *device, const void *_key) in _radv_create_vs_prolog() argument
158 return radv_create_vs_prolog(device, key); in _radv_create_vs_prolog()
184 radv_device_init_vs_prologs(struct radv_device *device) in radv_device_init_vs_prologs() argument
186 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_vs_prologs()
189 if (!radv_shader_part_cache_init(&device->vs_prologs, &vs_prolog_ops)) in radv_device_init_vs_prologs()
207 device->simple_vs_prologs[i - 1] = radv_create_vs_prolog(device, &key); in radv_device_init_vs_prologs()
208 if (!device->simple_vs_prologs[i - 1]) in radv_device_init_vs_prologs()
219 struct radv_shader_part *prolog = radv_create_vs_prolog(device, &key); in radv_device_init_vs_prologs()
224 device->instance_rate_vs_prologs[idx++] = prolog; in radv_device_init_vs_prologs()
228 assert(idx == ARRAY_SIZE(device->instance_rate_vs_prologs)); in radv_device_init_vs_prologs()
234 radv_device_finish_vs_prologs(struct radv_device *device) in radv_device_finish_vs_prologs() argument
236 if (device->vs_prologs.ops) in radv_device_finish_vs_prologs()
237 radv_shader_part_cache_finish(device, &device->vs_prologs); in radv_device_finish_vs_prologs()
239 for (unsigned i = 0; i < ARRAY_SIZE(device->simple_vs_prologs); i++) { in radv_device_finish_vs_prologs()
240 if (!device->simple_vs_prologs[i]) in radv_device_finish_vs_prologs()
243 radv_shader_part_unref(device, device->simple_vs_prologs[i]); in radv_device_finish_vs_prologs()
246 for (unsigned i = 0; i < ARRAY_SIZE(device->instance_rate_vs_prologs); i++) { in radv_device_finish_vs_prologs()
247 if (!device->instance_rate_vs_prologs[i]) in radv_device_finish_vs_prologs()
250 radv_shader_part_unref(device, device->instance_rate_vs_prologs[i]); in radv_device_finish_vs_prologs()
255 _radv_create_ps_epilog(struct radv_device *device, const void *_key) in _radv_create_ps_epilog() argument
258 return radv_create_ps_epilog(device, key, NULL); in _radv_create_ps_epilog()
284 radv_device_init_vrs_state(struct radv_device *device) in radv_device_init_vrs_state() argument
308 …radv_image_create(radv_device_to_handle(device), &(struct radv_image_create_info){.vk_info = &imag… in radv_device_init_vrs_state()
309 &device->meta_state.alloc, &image, true); in radv_device_init_vrs_state()
324 …result = radv_create_buffer(device, &buffer_create_info, &device->meta_state.alloc, &buffer, true); in radv_device_init_vrs_state()
335 vk_common_GetBufferMemoryRequirements2(radv_device_to_handle(device), &info, &mem_req); in radv_device_init_vrs_state()
342 result = radv_alloc_memory(device, &alloc_info, &device->meta_state.alloc, &mem, true); in radv_device_init_vrs_state()
351 result = radv_BindBufferMemory2(radv_device_to_handle(device), 1, &bind_info); in radv_device_init_vrs_state()
355 device->vrs.image = radv_image_from_handle(image); in radv_device_init_vrs_state()
356 device->vrs.buffer = radv_buffer_from_handle(buffer); in radv_device_init_vrs_state()
357 device->vrs.mem = radv_device_memory_from_handle(mem); in radv_device_init_vrs_state()
362 radv_FreeMemory(radv_device_to_handle(device), mem, &device->meta_state.alloc); in radv_device_init_vrs_state()
364 radv_DestroyBuffer(radv_device_to_handle(device), buffer, &device->meta_state.alloc); in radv_device_init_vrs_state()
366 radv_DestroyImage(radv_device_to_handle(device), image, &device->meta_state.alloc); in radv_device_init_vrs_state()
372 radv_device_finish_vrs_image(struct radv_device *device) in radv_device_finish_vrs_image() argument
374 if (!device->vrs.image) in radv_device_finish_vrs_image()
377 radv_FreeMemory(radv_device_to_handle(device), radv_device_memory_to_handle(device->vrs.mem), in radv_device_finish_vrs_image()
378 &device->meta_state.alloc); in radv_device_finish_vrs_image()
379 radv_DestroyBuffer(radv_device_to_handle(device), radv_buffer_to_handle(device->vrs.buffer), in radv_device_finish_vrs_image()
380 &device->meta_state.alloc); in radv_device_finish_vrs_image()
381 …radv_DestroyImage(radv_device_to_handle(device), radv_image_to_handle(device->vrs.image), &device-… in radv_device_finish_vrs_image()
436 struct radv_device *device = data; in radv_notifier_thread_run() local
437 struct radv_notifier *notifier = &device->notifier; in radv_notifier_thread_run()
453 device->force_vrs = radv_parse_force_vrs_config_file(file); in radv_notifier_thread_run()
455 fprintf(stderr, "radv: Updated the per-vertex VRS rate to '%d'.\n", device->force_vrs); in radv_notifier_thread_run()
473 radv_device_init_notifier(struct radv_device *device) in radv_device_init_notifier() argument
478 struct radv_notifier *notifier = &device->notifier; in radv_device_init_notifier()
490 ret = thrd_create(¬ifier->thread, radv_notifier_thread_run, device); in radv_device_init_notifier()
506 radv_device_finish_notifier(struct radv_device *device) in radv_device_finish_notifier() argument
509 struct radv_notifier *notifier = &device->notifier; in radv_device_finish_notifier()
522 radv_device_init_perf_counter(struct radv_device *device) in radv_device_init_perf_counter() argument
524 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_perf_counter()
528 result = radv_bo_create(device, NULL, bo_size, 4096, RADEON_DOMAIN_GTT, in radv_device_init_perf_counter()
530 0, true, &device->perf_counter_bo); in radv_device_init_perf_counter()
534 … device->perf_counter_lock_cs = calloc(sizeof(struct radeon_winsys_cs *), 2 * PERF_CTR_MAX_PASSES); in radv_device_init_perf_counter()
535 if (!device->perf_counter_lock_cs) in radv_device_init_perf_counter()
545 radv_device_finish_perf_counter(struct radv_device *device) in radv_device_finish_perf_counter() argument
547 if (device->perf_counter_bo) in radv_device_finish_perf_counter()
548 radv_bo_destroy(device, NULL, device->perf_counter_bo); in radv_device_finish_perf_counter()
550 if (!device->perf_counter_lock_cs) in radv_device_finish_perf_counter()
554 if (device->perf_counter_lock_cs[i]) in radv_device_finish_perf_counter()
555 device->ws->cs_destroy(device->perf_counter_lock_cs[i]); in radv_device_finish_perf_counter()
558 free(device->perf_counter_lock_cs); in radv_device_finish_perf_counter()
562 radv_device_init_memory_cache(struct radv_device *device) in radv_device_init_memory_cache() argument
566 device->mem_cache = vk_pipeline_cache_create(&device->vk, &info, NULL); in radv_device_init_memory_cache()
567 if (!device->mem_cache) in radv_device_init_memory_cache()
574 radv_device_finish_memory_cache(struct radv_device *device) in radv_device_finish_memory_cache() argument
576 if (device->mem_cache) in radv_device_finish_memory_cache()
577 vk_pipeline_cache_destroy(device->mem_cache, NULL); in radv_device_finish_memory_cache()
581 radv_device_init_rgp(struct radv_device *device) in radv_device_init_rgp() argument
583 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_rgp()
596 if (!radv_sqtt_init(device)) in radv_device_init_rgp()
602 …device->sqtt.buffer_size / (1024 * 1024), radv_is_instruction_timing_enabled() ? "enabled" : "disa… in radv_device_init_rgp()
608 if (!radv_spm_init(device)) in radv_device_init_rgp()
619 radv_device_finish_rgp(struct radv_device *device) in radv_device_finish_rgp() argument
621 radv_sqtt_finish(device); in radv_device_finish_rgp()
622 radv_spm_finish(device); in radv_device_finish_rgp()
626 radv_device_init_rmv(struct radv_device *device) in radv_device_init_rmv() argument
628 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_rmv()
637 vk_memory_trace_init(&device->vk, &info); in radv_device_init_rmv()
638 radv_memory_trace_init(device); in radv_device_init_rmv()
642 radv_device_init_trap_handler(struct radv_device *device) in radv_device_init_trap_handler() argument
644 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_trap_handler()
656 if (!radv_trap_handler_init(device)) in radv_device_init_trap_handler()
663 radv_device_init_device_fault_detection(struct radv_device *device) in radv_device_init_device_fault_detection() argument
665 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_device_fault_detection()
668 if (!radv_device_fault_detection_enabled(device)) in radv_device_init_device_fault_detection()
671 if (!radv_init_trace(device)) in radv_device_init_device_fault_detection()
683 radv_dump_enabled_options(device, stderr); in radv_device_init_device_fault_detection()
689 radv_device_finish_device_fault_detection(struct radv_device *device) in radv_device_finish_device_fault_detection() argument
691 radv_finish_trace(device); in radv_device_finish_device_fault_detection()
692 ralloc_free(device->gpu_hang_report); in radv_device_finish_device_fault_detection()
696 radv_device_init_tools(struct radv_device *device) in radv_device_init_tools() argument
698 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_tools()
702 result = radv_device_init_device_fault_detection(device); in radv_device_init_tools()
706 result = radv_device_init_rgp(device); in radv_device_init_tools()
710 radv_device_init_rmv(device); in radv_device_init_tools()
712 result = radv_device_init_trap_handler(device); in radv_device_init_tools()
717 result = radv_rra_trace_init(device); in radv_device_init_tools()
722 result = radv_printf_data_init(device); in radv_device_init_tools()
730 radv_device_finish_tools(struct radv_device *device) in radv_device_finish_tools() argument
732 radv_printf_data_finish(device); in radv_device_finish_tools()
733 radv_rra_trace_finish(radv_device_to_handle(device), &device->rra_trace); in radv_device_finish_tools()
734 radv_trap_handler_finish(device); in radv_device_finish_tools()
735 radv_memory_trace_finish(device); in radv_device_finish_tools()
736 radv_device_finish_rgp(device); in radv_device_finish_tools()
737 radv_device_finish_device_fault_detection(device); in radv_device_finish_tools()
762 init_dispatch_tables(struct radv_device *device, struct radv_physical_device *pdev) in init_dispatch_tables() argument
766 b.tables[RADV_DEVICE_DISPATCH_TABLE] = &device->vk.dispatch_table; in init_dispatch_tables()
767 b.tables[RADV_ANNOTATE_DISPATCH_TABLE] = &device->layer_dispatch.annotate; in init_dispatch_tables()
768 b.tables[RADV_APP_DISPATCH_TABLE] = &device->layer_dispatch.app; in init_dispatch_tables()
769 b.tables[RADV_RGP_DISPATCH_TABLE] = &device->layer_dispatch.rgp; in init_dispatch_tables()
770 b.tables[RADV_RRA_DISPATCH_TABLE] = &device->layer_dispatch.rra; in init_dispatch_tables()
771 b.tables[RADV_RMV_DISPATCH_TABLE] = &device->layer_dispatch.rmv; in init_dispatch_tables()
772 b.tables[RADV_CTX_ROLL_DISPATCH_TABLE] = &device->layer_dispatch.ctx_roll; in init_dispatch_tables()
775 if (radv_device_fault_detection_enabled(device) || gather_ctx_rolls) in init_dispatch_tables()
808 struct radv_device *device = container_of(_device, struct radv_device, vk); in get_timestamp() local
809 *timestamp = device->ws->query_value(device->ws, RADEON_TIMESTAMP); in get_timestamp()
817 struct radv_device *device = radv_queue_device(queue); in capture_trace() local
818 const struct radv_physical_device *pdev = radv_device_physical(device); in capture_trace()
824 device->rra_trace.triggered = true; in capture_trace()
826 if (device->vk.memory_trace_data.is_enabled) { in capture_trace()
827 simple_mtx_lock(&device->vk.memory_trace_data.token_mtx); in capture_trace()
828 radv_rmv_collect_trace_events(device); in capture_trace()
829 vk_dump_rmv_capture(&device->vk.memory_trace_data); in capture_trace()
830 simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx); in capture_trace()
834 device->sqtt_triggered = true; in capture_trace()
843 simple_mtx_lock(&device->ctx_roll_mtx); in capture_trace()
845 device->ctx_roll_file = fopen(filename, "w"); in capture_trace()
846 if (device->ctx_roll_file) in capture_trace()
849 simple_mtx_unlock(&device->ctx_roll_mtx); in capture_trace()
856 radv_device_init_cache_key(struct radv_device *device) in radv_device_init_cache_key() argument
858 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_init_cache_key()
860 struct radv_device_cache_key *key = &device->cache_key; in radv_device_init_cache_key()
862 key->keep_shader_info = device->keep_shader_info; in radv_device_init_cache_key()
863 key->trap_excp_flags = device->trap_handler_shader && instance->trap_excp_flags; in radv_device_init_cache_key()
864 key->disable_trunc_coord = device->disable_trunc_coord; in radv_device_init_cache_key()
865 …key->image_2d_view_of_3d = device->vk.enabled_features.image2DViewOf3D && pdev->info.gfx_level == … in radv_device_init_cache_key()
866 …key->mesh_shader_queries = device->vk.enabled_features.meshShaderQueries && pdev->emulate_mesh_sha… in radv_device_init_cache_key()
867 key->primitives_generated_query = radv_uses_primitives_generated_query(device); in radv_device_init_cache_key()
870 * "Binary shaders retrieved from a physical device with a certain shaderBinaryUUID are in radv_device_init_cache_key()
875 * enabled, regardless of what features are actually enabled on the logical device. in radv_device_init_cache_key()
877 if (device->vk.enabled_features.shaderObject) { in radv_device_init_cache_key()
882 _mesa_blake3_compute(key, sizeof(*key), device->cache_hash); in radv_device_init_cache_key()
886 radv_create_gfx_preamble(struct radv_device *device) in radv_create_gfx_preamble() argument
888 struct radeon_cmdbuf *cs = device->ws->cs_create(device->ws, AMD_IP_GFX, false); in radv_create_gfx_preamble()
892 radeon_check_space(device->ws, cs, 512); in radv_create_gfx_preamble()
894 radv_emit_graphics(device, cs); in radv_create_gfx_preamble()
896 device->ws->cs_pad(cs, 0); in radv_create_gfx_preamble()
899 device, NULL, cs->cdw * 4, 4096, device->ws->cs_domain(device->ws), in radv_create_gfx_preamble()
901 RADV_BO_PRIORITY_CS, 0, true, &device->gfx_init); in radv_create_gfx_preamble()
905 void *map = radv_buffer_map(device->ws, device->gfx_init); in radv_create_gfx_preamble()
907 radv_bo_destroy(device, NULL, device->gfx_init); in radv_create_gfx_preamble()
908 device->gfx_init = NULL; in radv_create_gfx_preamble()
913 device->ws->buffer_unmap(device->ws, device->gfx_init, false); in radv_create_gfx_preamble()
914 device->gfx_init_size_dw = cs->cdw; in radv_create_gfx_preamble()
916 device->ws->cs_destroy(cs); in radv_create_gfx_preamble()
1031 radv_get_sample_position(struct radv_device *device, unsigned sample_count, unsigned sample_index, … in radv_get_sample_position() argument
1056 radv_device_init_msaa(struct radv_device *device) in radv_device_init_msaa() argument
1060 radv_get_sample_position(device, 1, 0, device->sample_locations_1x[0]); in radv_device_init_msaa()
1063 radv_get_sample_position(device, 2, i, device->sample_locations_2x[i]); in radv_device_init_msaa()
1065 radv_get_sample_position(device, 4, i, device->sample_locations_4x[i]); in radv_device_init_msaa()
1067 radv_get_sample_position(device, 8, i, device->sample_locations_8x[i]); in radv_device_init_msaa()
1071 radv_destroy_device(struct radv_device *device, const VkAllocationCallbacks *pAllocator) in radv_destroy_device() argument
1073 radv_device_finish_perf_counter(device); in radv_destroy_device()
1075 if (device->gfx_init) in radv_destroy_device()
1076 radv_bo_destroy(device, NULL, device->gfx_init); in radv_destroy_device()
1078 radv_device_finish_notifier(device); in radv_destroy_device()
1079 radv_device_finish_vs_prologs(device); in radv_destroy_device()
1080 if (device->ps_epilogs.ops) in radv_destroy_device()
1081 radv_shader_part_cache_finish(device, &device->ps_epilogs); in radv_destroy_device()
1082 radv_device_finish_border_color(device); in radv_destroy_device()
1083 radv_device_finish_vrs_image(device); in radv_destroy_device()
1086 for (unsigned q = 0; q < device->queue_count[i]; q++) in radv_destroy_device()
1087 radv_queue_finish(&device->queues[i][q]); in radv_destroy_device()
1088 if (device->queue_count[i]) in radv_destroy_device()
1089 vk_free(&device->vk.alloc, device->queues[i]); in radv_destroy_device()
1091 if (device->private_sdma_queue != VK_NULL_HANDLE) { in radv_destroy_device()
1092 radv_queue_finish(device->private_sdma_queue); in radv_destroy_device()
1093 vk_free(&device->vk.alloc, device->private_sdma_queue); in radv_destroy_device()
1096 _mesa_hash_table_destroy(device->rt_handles, NULL); in radv_destroy_device()
1098 radv_device_finish_meta(device); in radv_destroy_device()
1099 radv_device_finish_tools(device); in radv_destroy_device()
1100 radv_device_finish_memory_cache(device); in radv_destroy_device()
1102 radv_destroy_shader_upload_queue(device); in radv_destroy_device()
1105 if (device->hw_ctx[i]) in radv_destroy_device()
1106 device->ws->ctx_destroy(device->hw_ctx[i]); in radv_destroy_device()
1109 mtx_destroy(&device->overallocation_mutex); in radv_destroy_device()
1110 simple_mtx_destroy(&device->ctx_roll_mtx); in radv_destroy_device()
1111 simple_mtx_destroy(&device->pstate_mtx); in radv_destroy_device()
1112 simple_mtx_destroy(&device->trace_mtx); in radv_destroy_device()
1113 simple_mtx_destroy(&device->rt_handles_mtx); in radv_destroy_device()
1114 simple_mtx_destroy(&device->pso_cache_stats_mtx); in radv_destroy_device()
1116 radv_destroy_shader_arenas(device); in radv_destroy_device()
1117 if (device->capture_replay_arena_vas) in radv_destroy_device()
1118 _mesa_hash_table_u64_destroy(device->capture_replay_arena_vas); in radv_destroy_device()
1120 vk_device_finish(&device->vk); in radv_destroy_device()
1121 vk_free(&device->vk.alloc, device); in radv_destroy_device()
1131 struct radv_device *device; in radv_CreateDevice() local
1148 …device = vk_zalloc2(&instance->vk.alloc, pAllocator, sizeof(*device), 8, VK_SYSTEM_ALLOCATION_SCOP… in radv_CreateDevice()
1149 if (!device) in radv_CreateDevice()
1152 result = vk_device_init(&device->vk, &pdev->vk, NULL, pCreateInfo, pAllocator); in radv_CreateDevice()
1154 vk_free(&device->vk.alloc, device); in radv_CreateDevice()
1158 device->vk.get_timestamp = get_timestamp; in radv_CreateDevice()
1159 device->vk.capture_trace = capture_trace; in radv_CreateDevice()
1161 device->vk.command_buffer_ops = &radv_cmd_buffer_ops; in radv_CreateDevice()
1163 init_dispatch_tables(device, pdev); in radv_CreateDevice()
1165 simple_mtx_init(&device->ctx_roll_mtx, mtx_plain); in radv_CreateDevice()
1166 simple_mtx_init(&device->trace_mtx, mtx_plain); in radv_CreateDevice()
1167 simple_mtx_init(&device->pstate_mtx, mtx_plain); in radv_CreateDevice()
1168 simple_mtx_init(&device->rt_handles_mtx, mtx_plain); in radv_CreateDevice()
1169 simple_mtx_init(&device->pso_cache_stats_mtx, mtx_plain); in radv_CreateDevice()
1171 device->rt_handles = _mesa_hash_table_create(NULL, _mesa_hash_u32, _mesa_key_u32_equal); in radv_CreateDevice()
1173 device->ws = pdev->ws; in radv_CreateDevice()
1174 vk_device_set_drm_fd(&device->vk, device->ws->get_fd(device->ws)); in radv_CreateDevice()
1179 device->use_global_bo_list = (instance->perftest_flags & RADV_PERFTEST_BO_LIST) || in radv_CreateDevice()
1180 device->vk.enabled_features.bufferDeviceAddress || in radv_CreateDevice()
1181 device->vk.enabled_features.descriptorIndexing || in radv_CreateDevice()
1182 … device->vk.enabled_features.descriptorBindingUniformBufferUpdateAfterBind || in radv_CreateDevice()
1183 … device->vk.enabled_features.descriptorBindingSampledImageUpdateAfterBind || in radv_CreateDevice()
1184 … device->vk.enabled_features.descriptorBindingStorageImageUpdateAfterBind || in radv_CreateDevice()
1185 … device->vk.enabled_features.descriptorBindingStorageBufferUpdateAfterBind || in radv_CreateDevice()
1186 … device->vk.enabled_features.descriptorBindingUniformTexelBufferUpdateAfterBind || in radv_CreateDevice()
1187 … device->vk.enabled_features.descriptorBindingStorageTexelBufferUpdateAfterBind || in radv_CreateDevice()
1188 … device->vk.enabled_features.descriptorBindingUpdateUnusedWhilePending || in radv_CreateDevice()
1189 device->vk.enabled_features.descriptorBindingPartiallyBound; in radv_CreateDevice()
1191 radv_init_shader_arenas(device); in radv_CreateDevice()
1193 device->overallocation_disallowed = overallocation_disallowed; in radv_CreateDevice()
1194 mtx_init(&device->overallocation_mutex, mtx_plain); in radv_CreateDevice()
1197 device->uses_shadow_regs = true; in radv_CreateDevice()
1206 if (device->hw_ctx[priority]) in radv_CreateDevice()
1209 result = device->ws->ctx_create(device->ws, priority, &device->hw_ctx[priority]); in radv_CreateDevice()
1220 …device->queues[qfi] = vk_zalloc(&device->vk.alloc, queue_create->queueCount * sizeof(struct radv_q… in radv_CreateDevice()
1222 if (!device->queues[qfi]) { in radv_CreateDevice()
1227 device->queue_count[qfi] = queue_create->queueCount; in radv_CreateDevice()
1230 … result = radv_queue_init(device, &device->queues[qfi][q], q, queue_create, global_priority); in radv_CreateDevice()
1235 device->private_sdma_queue = VK_NULL_HANDLE; in radv_CreateDevice()
1237 device->shader_use_invisible_vram = (instance->perftest_flags & RADV_PERFTEST_DMA_SHADERS) && in radv_CreateDevice()
1240 result = radv_init_shader_upload_queue(device); in radv_CreateDevice()
1244 …device->pbb_allowed = pdev->info.gfx_level >= GFX9 && !(instance->debug_flags & RADV_DEBUG_NOBINNI… in radv_CreateDevice()
1246 device->disable_trunc_coord = instance->drirc.disable_trunc_coord; in radv_CreateDevice()
1250 bool is_d3d9 = !device->vk.enabled_features.dualSrcBlend; in radv_CreateDevice()
1254 device->disable_trunc_coord &= !is_d3d9; in radv_CreateDevice()
1270 device->scratch_waves = MAX2(32 * pdev->info.num_cu, max_threads_per_block / 64); in radv_CreateDevice()
1272 device->dispatch_initiator = S_00B800_COMPUTE_SHADER_EN(1); in radv_CreateDevice()
1278 device->dispatch_initiator |= S_00B800_ORDER_MODE(1); in radv_CreateDevice()
1285 device->dispatch_initiator |= S_00B800_TUNNEL_ENABLE(1); in radv_CreateDevice()
1292 … device->dispatch_initiator_task = device->dispatch_initiator | S_00B800_DISABLE_DISP_PREMPT_EN(1); in radv_CreateDevice()
1298 device->force_vrs = radv_parse_force_vrs_config_file(file); in radv_CreateDevice()
1300 if (radv_device_init_notifier(device)) { in radv_CreateDevice()
1301 device->force_vrs_enabled = true; in radv_CreateDevice()
1308 device->force_vrs = radv_parse_vrs_rates(vrs_rates); in radv_CreateDevice()
1309 device->force_vrs_enabled = device->force_vrs != RADV_FORCE_VRS_1x1; in radv_CreateDevice()
1314 device->load_grid_size_from_user_sgpr = pdev->info.gfx_level >= GFX10_3; in radv_CreateDevice()
1317 …device->keep_shader_info = radv_device_fault_detection_enabled(device) || radv_trap_handler_enable… in radv_CreateDevice()
1319 /* Initialize the per-device cache key before compiling meta shaders. */ in radv_CreateDevice()
1320 radv_device_init_cache_key(device); in radv_CreateDevice()
1322 result = radv_device_init_tools(device); in radv_CreateDevice()
1326 result = radv_device_init_meta(device); in radv_CreateDevice()
1330 radv_device_init_msaa(device); in radv_CreateDevice()
1333 if (device->vk.enabled_features.customBorderColors) { in radv_CreateDevice()
1334 result = radv_device_init_border_color(device); in radv_CreateDevice()
1339 …if (device->vk.enabled_features.vertexInputDynamicState || device->vk.enabled_features.graphicsPip… in radv_CreateDevice()
1340 device->vk.enabled_features.shaderObject) { in radv_CreateDevice()
1341 result = radv_device_init_vs_prologs(device); in radv_CreateDevice()
1346 …if (device->vk.enabled_features.graphicsPipelineLibrary || device->vk.enabled_features.shaderObjec… in radv_CreateDevice()
1347 device->vk.enabled_features.extendedDynamicState3ColorBlendEnable || in radv_CreateDevice()
1348 device->vk.enabled_features.extendedDynamicState3ColorWriteMask || in radv_CreateDevice()
1349 device->vk.enabled_features.extendedDynamicState3AlphaToCoverageEnable || in radv_CreateDevice()
1350 device->vk.enabled_features.extendedDynamicState3ColorBlendEquation) { in radv_CreateDevice()
1351 if (!radv_shader_part_cache_init(&device->ps_epilogs, &ps_epilog_ops)) { in radv_CreateDevice()
1358 radv_create_gfx_preamble(device); in radv_CreateDevice()
1360 if (!device->vk.disable_internal_cache) { in radv_CreateDevice()
1361 result = radv_device_init_memory_cache(device); in radv_CreateDevice()
1366 device->force_aniso = MIN2(16, (int)debug_get_num_option("RADV_TEX_ANISO", -1)); in radv_CreateDevice()
1367 if (device->force_aniso >= 0) { in radv_CreateDevice()
1368 …fprintf(stderr, "radv: Forcing anisotropy filter to %ix\n", 1 << util_logbase2(device->force_aniso… in radv_CreateDevice()
1371 if (device->vk.enabled_features.performanceCounterQueryPools) { in radv_CreateDevice()
1372 result = radv_device_init_perf_counter(device); in radv_CreateDevice()
1377 if (device->vk.enabled_features.rayTracingPipelineShaderGroupHandleCaptureReplay) { in radv_CreateDevice()
1378 device->capture_replay_arena_vas = _mesa_hash_table_u64_create(NULL); in radv_CreateDevice()
1382 if (!radv_device_acquire_performance_counters(device)) in radv_CreateDevice()
1386 *pDevice = radv_device_to_handle(device); in radv_CreateDevice()
1390 radv_destroy_device(device, pAllocator); in radv_CreateDevice()
1397 VK_FROM_HANDLE(radv_device, device, _device); in radv_DestroyDevice()
1399 if (!device) in radv_DestroyDevice()
1402 radv_destroy_device(device, pAllocator); in radv_DestroyDevice()
1409 VK_FROM_HANDLE(radv_device, device, _device); in radv_GetImageMemoryRequirements2()
1411 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_GetImageMemoryRequirements2()
1449 radv_GetDeviceImageMemoryRequirements(VkDevice device, const VkDeviceImageMemoryRequirements *pInfo, in radv_GetDeviceImageMemoryRequirements() argument
1460 …radv_image_create(device, &(struct radv_image_create_info){.vk_info = pInfo->pCreateInfo}, NULL, &… in radv_GetDeviceImageMemoryRequirements()
1468 radv_GetImageMemoryRequirements2(device, &info2, pMemoryRequirements); in radv_GetDeviceImageMemoryRequirements()
1470 radv_DestroyImage(device, image, NULL); in radv_GetDeviceImageMemoryRequirements()
1481 radv_get_dcc_max_uncompressed_block_size(const struct radv_device *device, const struct radv_image … in radv_get_dcc_max_uncompressed_block_size() argument
1483 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_get_dcc_max_uncompressed_block_size()
1496 radv_initialise_color_surface(struct radv_device *device, struct radv_color_buffer_info *cb, in radv_initialise_color_surface() argument
1499 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_initialise_color_surface()
1574 radv_initialise_ds_surface(const struct radv_device *device, struct radv_ds_buffer_info *ds, in radv_initialise_ds_surface() argument
1577 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_initialise_ds_surface()
1606 .htile_stencil_disabled = radv_image_tile_stencil_disabled(device, iview->image), in radv_initialise_ds_surface()
1607 .vrs_enabled = radv_image_has_vrs_htile(device, iview->image), in radv_initialise_ds_surface()
1623 radv_gfx11_set_db_render_control(device, iview->image->vk.samples, &ds->db_render_control); in radv_initialise_ds_surface()
1628 radv_gfx11_set_db_render_control(const struct radv_device *device, unsigned num_samples, unsigned *… in radv_gfx11_set_db_render_control() argument
1630 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_gfx11_set_db_render_control()
1655 VK_FROM_HANDLE(radv_device, device, _device); in radv_GetMemoryFdKHR()
1671 radv_image_bo_set_metadata(device, image, memory->bo); in radv_GetMemoryFdKHR()
1674 bool ret = device->ws->buffer_get_fd(device->ws, memory->bo, pFD); in radv_GetMemoryFdKHR()
1676 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); in radv_GetMemoryFdKHR()
1730 VK_FROM_HANDLE(radv_device, device, _device); in radv_GetMemoryFdPropertiesKHR()
1731 struct radv_physical_device *pdev = radv_device_physical(device); in radv_GetMemoryFdPropertiesKHR()
1737 if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags)) in radv_GetMemoryFdPropertiesKHR()
1738 return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE); in radv_GetMemoryFdPropertiesKHR()
1751 return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE); in radv_GetMemoryFdPropertiesKHR()
1756 radv_device_set_pstate(struct radv_device *device, bool enable) in radv_device_set_pstate() argument
1758 const struct radv_physical_device *pdev = radv_device_physical(device); in radv_device_set_pstate()
1760 struct radeon_winsys *ws = device->ws; in radv_device_set_pstate()
1764 /* pstate is per-device; setting it for one ctx is sufficient. in radv_device_set_pstate()
1767 if (device->hw_ctx[i]) in radv_device_set_pstate()
1768 return ws->ctx_set_pstate(device->hw_ctx[i], pstate) >= 0; in radv_device_set_pstate()
1775 radv_device_acquire_performance_counters(struct radv_device *device) in radv_device_acquire_performance_counters() argument
1778 simple_mtx_lock(&device->pstate_mtx); in radv_device_acquire_performance_counters()
1780 if (device->pstate_cnt == 0) { in radv_device_acquire_performance_counters()
1781 result = radv_device_set_pstate(device, true); in radv_device_acquire_performance_counters()
1783 ++device->pstate_cnt; in radv_device_acquire_performance_counters()
1786 simple_mtx_unlock(&device->pstate_mtx); in radv_device_acquire_performance_counters()
1791 radv_device_release_performance_counters(struct radv_device *device) in radv_device_release_performance_counters() argument
1793 simple_mtx_lock(&device->pstate_mtx); in radv_device_release_performance_counters()
1795 if (--device->pstate_cnt == 0) in radv_device_release_performance_counters()
1796 radv_device_set_pstate(device, false); in radv_device_release_performance_counters()
1798 simple_mtx_unlock(&device->pstate_mtx); in radv_device_release_performance_counters()
1804 VK_FROM_HANDLE(radv_device, device, _device); in radv_AcquireProfilingLockKHR()
1805 bool result = radv_device_acquire_performance_counters(device); in radv_AcquireProfilingLockKHR()
1812 VK_FROM_HANDLE(radv_device, device, _device); in radv_ReleaseProfilingLockKHR()
1813 radv_device_release_performance_counters(device); in radv_ReleaseProfilingLockKHR()
1817 radv_GetDeviceImageSubresourceLayout(VkDevice device, const VkDeviceImageSubresourceInfo *pInfo, in radv_GetDeviceImageSubresourceLayout() argument
1824 …radv_image_create(device, &(struct radv_image_create_info){.vk_info = pInfo->pCreateInfo}, NULL, &… in radv_GetDeviceImageSubresourceLayout()
1827 radv_GetImageSubresourceLayout2(device, image, pInfo->pSubresource, pLayout); in radv_GetDeviceImageSubresourceLayout()
1829 radv_DestroyImage(device, image, NULL); in radv_GetDeviceImageSubresourceLayout()