1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_pipe.h"
25 #include "si_public.h"
26 #include "si_shader_internal.h"
27 #include "sid.h"
28
29 #include "radeon/r600_cs.h"
30 #include "radeon/radeon_uvd.h"
31 #include "util/hash_table.h"
32 #include "util/u_log.h"
33 #include "util/u_memory.h"
34 #include "util/u_suballoc.h"
35 #include "util/u_tests.h"
36 #include "util/xmlconfig.h"
37 #include "vl/vl_decoder.h"
38 #include "../ddebug/dd_util.h"
39
40 static const struct debug_named_value debug_options[] = {
41 /* Shader logging options: */
42 { "vs", DBG(VS), "Print vertex shaders" },
43 { "ps", DBG(PS), "Print pixel shaders" },
44 { "gs", DBG(GS), "Print geometry shaders" },
45 { "tcs", DBG(TCS), "Print tessellation control shaders" },
46 { "tes", DBG(TES), "Print tessellation evaluation shaders" },
47 { "cs", DBG(CS), "Print compute shaders" },
48 { "noir", DBG(NO_IR), "Don't print the LLVM IR"},
49 { "notgsi", DBG(NO_TGSI), "Don't print the TGSI"},
50 { "noasm", DBG(NO_ASM), "Don't print disassembled shaders"},
51 { "preoptir", DBG(PREOPT_IR), "Print the LLVM IR before initial optimizations" },
52
53 /* Shader compiler options the shader cache should be aware of: */
54 { "unsafemath", DBG(UNSAFE_MATH), "Enable unsafe math shader optimizations" },
55 { "sisched", DBG(SI_SCHED), "Enable LLVM SI Machine Instruction Scheduler." },
56
57 /* Shader compiler options (with no effect on the shader cache): */
58 { "checkir", DBG(CHECK_IR), "Enable additional sanity checks on shader IR" },
59 { "precompile", DBG(PRECOMPILE), "Compile one shader variant at shader creation." },
60 { "nir", DBG(NIR), "Enable experimental NIR shaders" },
61 { "mono", DBG(MONOLITHIC_SHADERS), "Use old-style monolithic shaders compiled on demand" },
62 { "nooptvariant", DBG(NO_OPT_VARIANT), "Disable compiling optimized shader variants." },
63
64 /* Information logging options: */
65 { "info", DBG(INFO), "Print driver information" },
66 { "tex", DBG(TEX), "Print texture info" },
67 { "compute", DBG(COMPUTE), "Print compute info" },
68 { "vm", DBG(VM), "Print virtual addresses when creating resources" },
69
70 /* Driver options: */
71 { "forcedma", DBG(FORCE_DMA), "Use asynchronous DMA for all operations when possible." },
72 { "nodma", DBG(NO_ASYNC_DMA), "Disable asynchronous DMA" },
73 { "nowc", DBG(NO_WC), "Disable GTT write combining" },
74 { "check_vm", DBG(CHECK_VM), "Check VM faults and dump debug info." },
75 { "reserve_vmid", DBG(RESERVE_VMID), "Force VMID reservation per context." },
76
77 /* 3D engine options: */
78 { "switch_on_eop", DBG(SWITCH_ON_EOP), "Program WD/IA to switch on end-of-packet." },
79 { "nooutoforder", DBG(NO_OUT_OF_ORDER), "Disable out-of-order rasterization" },
80 { "nodpbb", DBG(NO_DPBB), "Disable DPBB." },
81 { "nodfsm", DBG(NO_DFSM), "Disable DFSM." },
82 { "dpbb", DBG(DPBB), "Enable DPBB." },
83 { "dfsm", DBG(DFSM), "Enable DFSM." },
84 { "nohyperz", DBG(NO_HYPERZ), "Disable Hyper-Z" },
85 { "norbplus", DBG(NO_RB_PLUS), "Disable RB+." },
86 { "no2d", DBG(NO_2D_TILING), "Disable 2D tiling" },
87 { "notiling", DBG(NO_TILING), "Disable tiling" },
88 { "nodcc", DBG(NO_DCC), "Disable DCC." },
89 { "nodccclear", DBG(NO_DCC_CLEAR), "Disable DCC fast clear." },
90 { "nodccfb", DBG(NO_DCC_FB), "Disable separate DCC on the main framebuffer" },
91 { "nodccmsaa", DBG(NO_DCC_MSAA), "Disable DCC for MSAA" },
92 { "dccmsaa", DBG(DCC_MSAA), "Enable DCC for MSAA" },
93
94 /* Tests: */
95 { "testdma", DBG(TEST_DMA), "Invoke SDMA tests and exit." },
96 { "testvmfaultcp", DBG(TEST_VMFAULT_CP), "Invoke a CP VM fault test and exit." },
97 { "testvmfaultsdma", DBG(TEST_VMFAULT_SDMA), "Invoke a SDMA VM fault test and exit." },
98 { "testvmfaultshader", DBG(TEST_VMFAULT_SHADER), "Invoke a shader VM fault test and exit." },
99
100 DEBUG_NAMED_VALUE_END /* must be last */
101 };
102
103 /*
104 * pipe_context
105 */
si_destroy_context(struct pipe_context * context)106 static void si_destroy_context(struct pipe_context *context)
107 {
108 struct si_context *sctx = (struct si_context *)context;
109 int i;
110
111 /* Unreference the framebuffer normally to disable related logic
112 * properly.
113 */
114 struct pipe_framebuffer_state fb = {};
115 if (context->set_framebuffer_state)
116 context->set_framebuffer_state(context, &fb);
117
118 si_release_all_descriptors(sctx);
119
120 pipe_resource_reference(&sctx->esgs_ring, NULL);
121 pipe_resource_reference(&sctx->gsvs_ring, NULL);
122 pipe_resource_reference(&sctx->tf_ring, NULL);
123 pipe_resource_reference(&sctx->tess_offchip_ring, NULL);
124 pipe_resource_reference(&sctx->null_const_buf.buffer, NULL);
125 r600_resource_reference(&sctx->border_color_buffer, NULL);
126 free(sctx->border_color_table);
127 r600_resource_reference(&sctx->scratch_buffer, NULL);
128 r600_resource_reference(&sctx->compute_scratch_buffer, NULL);
129 r600_resource_reference(&sctx->wait_mem_scratch, NULL);
130
131 si_pm4_free_state(sctx, sctx->init_config, ~0);
132 if (sctx->init_config_gs_rings)
133 si_pm4_free_state(sctx, sctx->init_config_gs_rings, ~0);
134 for (i = 0; i < ARRAY_SIZE(sctx->vgt_shader_config); i++)
135 si_pm4_delete_state(sctx, vgt_shader_config, sctx->vgt_shader_config[i]);
136
137 if (sctx->fixed_func_tcs_shader.cso)
138 sctx->b.b.delete_tcs_state(&sctx->b.b, sctx->fixed_func_tcs_shader.cso);
139 if (sctx->custom_dsa_flush)
140 sctx->b.b.delete_depth_stencil_alpha_state(&sctx->b.b, sctx->custom_dsa_flush);
141 if (sctx->custom_blend_resolve)
142 sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_resolve);
143 if (sctx->custom_blend_fmask_decompress)
144 sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_fmask_decompress);
145 if (sctx->custom_blend_eliminate_fastclear)
146 sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_eliminate_fastclear);
147 if (sctx->custom_blend_dcc_decompress)
148 sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_dcc_decompress);
149 if (sctx->vs_blit_pos)
150 sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_pos);
151 if (sctx->vs_blit_pos_layered)
152 sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_pos_layered);
153 if (sctx->vs_blit_color)
154 sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_color);
155 if (sctx->vs_blit_color_layered)
156 sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_color_layered);
157 if (sctx->vs_blit_texcoord)
158 sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_texcoord);
159
160 if (sctx->blitter)
161 util_blitter_destroy(sctx->blitter);
162
163 si_common_context_cleanup(&sctx->b);
164
165 LLVMDisposeTargetMachine(sctx->tm);
166
167 si_saved_cs_reference(&sctx->current_saved_cs, NULL);
168
169 _mesa_hash_table_destroy(sctx->tex_handles, NULL);
170 _mesa_hash_table_destroy(sctx->img_handles, NULL);
171
172 util_dynarray_fini(&sctx->resident_tex_handles);
173 util_dynarray_fini(&sctx->resident_img_handles);
174 util_dynarray_fini(&sctx->resident_tex_needs_color_decompress);
175 util_dynarray_fini(&sctx->resident_img_needs_color_decompress);
176 util_dynarray_fini(&sctx->resident_tex_needs_depth_decompress);
177 FREE(sctx);
178 }
179
180 static enum pipe_reset_status
si_amdgpu_get_reset_status(struct pipe_context * ctx)181 si_amdgpu_get_reset_status(struct pipe_context *ctx)
182 {
183 struct si_context *sctx = (struct si_context *)ctx;
184
185 return sctx->b.ws->ctx_query_reset_status(sctx->b.ctx);
186 }
187
188 /* Apitrace profiling:
189 * 1) qapitrace : Tools -> Profile: Measure CPU & GPU times
190 * 2) In the middle panel, zoom in (mouse wheel) on some bad draw call
191 * and remember its number.
192 * 3) In Mesa, enable queries and performance counters around that draw
193 * call and print the results.
194 * 4) glretrace --benchmark --markers ..
195 */
si_emit_string_marker(struct pipe_context * ctx,const char * string,int len)196 static void si_emit_string_marker(struct pipe_context *ctx,
197 const char *string, int len)
198 {
199 struct si_context *sctx = (struct si_context *)ctx;
200
201 dd_parse_apitrace_marker(string, len, &sctx->apitrace_call_number);
202
203 if (sctx->b.log)
204 u_log_printf(sctx->b.log, "\nString marker: %*s\n", len, string);
205 }
206
207 static LLVMTargetMachineRef
si_create_llvm_target_machine(struct si_screen * sscreen)208 si_create_llvm_target_machine(struct si_screen *sscreen)
209 {
210 enum ac_target_machine_options tm_options =
211 (sscreen->debug_flags & DBG(SI_SCHED) ? AC_TM_SISCHED : 0) |
212 (sscreen->info.chip_class >= GFX9 ? AC_TM_FORCE_ENABLE_XNACK : 0) |
213 (sscreen->info.chip_class < GFX9 ? AC_TM_FORCE_DISABLE_XNACK : 0) |
214 (!sscreen->llvm_has_working_vgpr_indexing ? AC_TM_PROMOTE_ALLOCA_TO_SCRATCH : 0);
215
216 return ac_create_target_machine(sscreen->info.family, tm_options);
217 }
218
si_set_debug_callback(struct pipe_context * ctx,const struct pipe_debug_callback * cb)219 static void si_set_debug_callback(struct pipe_context *ctx,
220 const struct pipe_debug_callback *cb)
221 {
222 struct si_context *sctx = (struct si_context *)ctx;
223 struct si_screen *screen = sctx->screen;
224
225 util_queue_finish(&screen->shader_compiler_queue);
226 util_queue_finish(&screen->shader_compiler_queue_low_priority);
227
228 if (cb)
229 sctx->debug = *cb;
230 else
231 memset(&sctx->debug, 0, sizeof(sctx->debug));
232 }
233
si_set_log_context(struct pipe_context * ctx,struct u_log_context * log)234 static void si_set_log_context(struct pipe_context *ctx,
235 struct u_log_context *log)
236 {
237 struct si_context *sctx = (struct si_context *)ctx;
238 sctx->b.log = log;
239
240 if (log)
241 u_log_add_auto_logger(log, si_auto_log_cs, sctx);
242 }
243
si_create_context(struct pipe_screen * screen,unsigned flags)244 static struct pipe_context *si_create_context(struct pipe_screen *screen,
245 unsigned flags)
246 {
247 struct si_context *sctx = CALLOC_STRUCT(si_context);
248 struct si_screen* sscreen = (struct si_screen *)screen;
249 struct radeon_winsys *ws = sscreen->ws;
250 int shader, i;
251
252 if (!sctx)
253 return NULL;
254
255 if (flags & PIPE_CONTEXT_DEBUG)
256 sscreen->record_llvm_ir = true; /* racy but not critical */
257
258 sctx->b.b.screen = screen; /* this must be set first */
259 sctx->b.b.priv = NULL;
260 sctx->b.b.destroy = si_destroy_context;
261 sctx->b.b.emit_string_marker = si_emit_string_marker;
262 sctx->b.b.set_debug_callback = si_set_debug_callback;
263 sctx->b.b.set_log_context = si_set_log_context;
264 sctx->b.set_atom_dirty = (void *)si_set_atom_dirty;
265 sctx->screen = sscreen; /* Easy accessing of screen/winsys. */
266 sctx->is_debug = (flags & PIPE_CONTEXT_DEBUG) != 0;
267
268 if (!si_common_context_init(&sctx->b, sscreen, flags))
269 goto fail;
270
271 if (sscreen->info.drm_major == 3)
272 sctx->b.b.get_device_reset_status = si_amdgpu_get_reset_status;
273
274 si_init_buffer_functions(sctx);
275 si_init_clear_functions(sctx);
276 si_init_blit_functions(sctx);
277 si_init_compute_functions(sctx);
278 si_init_cp_dma_functions(sctx);
279 si_init_debug_functions(sctx);
280 si_init_msaa_functions(sctx);
281 si_init_streamout_functions(sctx);
282
283 if (sscreen->info.has_hw_decode) {
284 sctx->b.b.create_video_codec = si_uvd_create_decoder;
285 sctx->b.b.create_video_buffer = si_video_buffer_create;
286 } else {
287 sctx->b.b.create_video_codec = vl_create_decoder;
288 sctx->b.b.create_video_buffer = vl_video_buffer_create;
289 }
290
291 sctx->b.gfx.cs = ws->cs_create(sctx->b.ctx, RING_GFX,
292 si_context_gfx_flush, sctx);
293 sctx->b.gfx.flush = si_context_gfx_flush;
294
295 /* Border colors. */
296 sctx->border_color_table = malloc(SI_MAX_BORDER_COLORS *
297 sizeof(*sctx->border_color_table));
298 if (!sctx->border_color_table)
299 goto fail;
300
301 sctx->border_color_buffer = (struct r600_resource*)
302 pipe_buffer_create(screen, 0, PIPE_USAGE_DEFAULT,
303 SI_MAX_BORDER_COLORS *
304 sizeof(*sctx->border_color_table));
305 if (!sctx->border_color_buffer)
306 goto fail;
307
308 sctx->border_color_map =
309 ws->buffer_map(sctx->border_color_buffer->buf,
310 NULL, PIPE_TRANSFER_WRITE);
311 if (!sctx->border_color_map)
312 goto fail;
313
314 si_init_all_descriptors(sctx);
315 si_init_fence_functions(sctx);
316 si_init_state_functions(sctx);
317 si_init_shader_functions(sctx);
318 si_init_viewport_functions(sctx);
319 si_init_ia_multi_vgt_param_table(sctx);
320
321 if (sctx->b.chip_class >= CIK)
322 cik_init_sdma_functions(sctx);
323 else
324 si_init_dma_functions(sctx);
325
326 if (sscreen->debug_flags & DBG(FORCE_DMA))
327 sctx->b.b.resource_copy_region = sctx->b.dma_copy;
328
329 sctx->blitter = util_blitter_create(&sctx->b.b);
330 if (sctx->blitter == NULL)
331 goto fail;
332 sctx->blitter->draw_rectangle = si_draw_rectangle;
333 sctx->blitter->skip_viewport_restore = true;
334
335 sctx->sample_mask.sample_mask = 0xffff;
336
337 if (sctx->b.chip_class >= GFX9) {
338 sctx->wait_mem_scratch = (struct r600_resource*)
339 pipe_buffer_create(screen, 0, PIPE_USAGE_DEFAULT, 4);
340 if (!sctx->wait_mem_scratch)
341 goto fail;
342
343 /* Initialize the memory. */
344 struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
345 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
346 radeon_emit(cs, S_370_DST_SEL(V_370_MEMORY_SYNC) |
347 S_370_WR_CONFIRM(1) |
348 S_370_ENGINE_SEL(V_370_ME));
349 radeon_emit(cs, sctx->wait_mem_scratch->gpu_address);
350 radeon_emit(cs, sctx->wait_mem_scratch->gpu_address >> 32);
351 radeon_emit(cs, sctx->wait_mem_number);
352 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
353 sctx->wait_mem_scratch,
354 RADEON_USAGE_WRITE, RADEON_PRIO_FENCE);
355 }
356
357 /* CIK cannot unbind a constant buffer (S_BUFFER_LOAD doesn't skip loads
358 * if NUM_RECORDS == 0). We need to use a dummy buffer instead. */
359 if (sctx->b.chip_class == CIK) {
360 sctx->null_const_buf.buffer =
361 si_aligned_buffer_create(screen,
362 R600_RESOURCE_FLAG_UNMAPPABLE,
363 PIPE_USAGE_DEFAULT, 16,
364 sctx->screen->info.tcc_cache_line_size);
365 if (!sctx->null_const_buf.buffer)
366 goto fail;
367 sctx->null_const_buf.buffer_size = sctx->null_const_buf.buffer->width0;
368
369 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
370 for (i = 0; i < SI_NUM_CONST_BUFFERS; i++) {
371 sctx->b.b.set_constant_buffer(&sctx->b.b, shader, i,
372 &sctx->null_const_buf);
373 }
374 }
375
376 si_set_rw_buffer(sctx, SI_HS_CONST_DEFAULT_TESS_LEVELS,
377 &sctx->null_const_buf);
378 si_set_rw_buffer(sctx, SI_VS_CONST_INSTANCE_DIVISORS,
379 &sctx->null_const_buf);
380 si_set_rw_buffer(sctx, SI_VS_CONST_CLIP_PLANES,
381 &sctx->null_const_buf);
382 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE,
383 &sctx->null_const_buf);
384 si_set_rw_buffer(sctx, SI_PS_CONST_SAMPLE_POSITIONS,
385 &sctx->null_const_buf);
386
387 /* Clear the NULL constant buffer, because loads should return zeros. */
388 si_clear_buffer(&sctx->b.b, sctx->null_const_buf.buffer, 0,
389 sctx->null_const_buf.buffer->width0, 0,
390 R600_COHERENCY_SHADER);
391 }
392
393 uint64_t max_threads_per_block;
394 screen->get_compute_param(screen, PIPE_SHADER_IR_TGSI,
395 PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK,
396 &max_threads_per_block);
397
398 /* The maximum number of scratch waves. Scratch space isn't divided
399 * evenly between CUs. The number is only a function of the number of CUs.
400 * We can decrease the constant to decrease the scratch buffer size.
401 *
402 * sctx->scratch_waves must be >= the maximum posible size of
403 * 1 threadgroup, so that the hw doesn't hang from being unable
404 * to start any.
405 *
406 * The recommended value is 4 per CU at most. Higher numbers don't
407 * bring much benefit, but they still occupy chip resources (think
408 * async compute). I've seen ~2% performance difference between 4 and 32.
409 */
410 sctx->scratch_waves = MAX2(32 * sscreen->info.num_good_compute_units,
411 max_threads_per_block / 64);
412
413 sctx->tm = si_create_llvm_target_machine(sscreen);
414
415 /* Bindless handles. */
416 sctx->tex_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
417 _mesa_key_pointer_equal);
418 sctx->img_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
419 _mesa_key_pointer_equal);
420
421 util_dynarray_init(&sctx->resident_tex_handles, NULL);
422 util_dynarray_init(&sctx->resident_img_handles, NULL);
423 util_dynarray_init(&sctx->resident_tex_needs_color_decompress, NULL);
424 util_dynarray_init(&sctx->resident_img_needs_color_decompress, NULL);
425 util_dynarray_init(&sctx->resident_tex_needs_depth_decompress, NULL);
426
427 /* this must be last */
428 si_begin_new_cs(sctx);
429 return &sctx->b.b;
430 fail:
431 fprintf(stderr, "radeonsi: Failed to create a context.\n");
432 si_destroy_context(&sctx->b.b);
433 return NULL;
434 }
435
si_pipe_create_context(struct pipe_screen * screen,void * priv,unsigned flags)436 static struct pipe_context *si_pipe_create_context(struct pipe_screen *screen,
437 void *priv, unsigned flags)
438 {
439 struct si_screen *sscreen = (struct si_screen *)screen;
440 struct pipe_context *ctx;
441
442 if (sscreen->debug_flags & DBG(CHECK_VM))
443 flags |= PIPE_CONTEXT_DEBUG;
444
445 ctx = si_create_context(screen, flags);
446
447 if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
448 return ctx;
449
450 /* Clover (compute-only) is unsupported. */
451 if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
452 return ctx;
453
454 /* When shaders are logged to stderr, asynchronous compilation is
455 * disabled too. */
456 if (sscreen->debug_flags & DBG_ALL_SHADERS)
457 return ctx;
458
459 /* Use asynchronous flushes only on amdgpu, since the radeon
460 * implementation for fence_server_sync is incomplete. */
461 return threaded_context_create(ctx, &sscreen->pool_transfers,
462 si_replace_buffer_storage,
463 sscreen->info.drm_major >= 3 ? si_create_fence : NULL,
464 &((struct si_context*)ctx)->b.tc);
465 }
466
467 /*
468 * pipe_screen
469 */
470
si_destroy_screen(struct pipe_screen * pscreen)471 static void si_destroy_screen(struct pipe_screen* pscreen)
472 {
473 struct si_screen *sscreen = (struct si_screen *)pscreen;
474 struct si_shader_part *parts[] = {
475 sscreen->vs_prologs,
476 sscreen->tcs_epilogs,
477 sscreen->gs_prologs,
478 sscreen->ps_prologs,
479 sscreen->ps_epilogs
480 };
481 unsigned i;
482
483 if (!sscreen->ws->unref(sscreen->ws))
484 return;
485
486 util_queue_destroy(&sscreen->shader_compiler_queue);
487 util_queue_destroy(&sscreen->shader_compiler_queue_low_priority);
488
489 for (i = 0; i < ARRAY_SIZE(sscreen->tm); i++)
490 if (sscreen->tm[i])
491 LLVMDisposeTargetMachine(sscreen->tm[i]);
492
493 for (i = 0; i < ARRAY_SIZE(sscreen->tm_low_priority); i++)
494 if (sscreen->tm_low_priority[i])
495 LLVMDisposeTargetMachine(sscreen->tm_low_priority[i]);
496
497 /* Free shader parts. */
498 for (i = 0; i < ARRAY_SIZE(parts); i++) {
499 while (parts[i]) {
500 struct si_shader_part *part = parts[i];
501
502 parts[i] = part->next;
503 ac_shader_binary_clean(&part->binary);
504 FREE(part);
505 }
506 }
507 mtx_destroy(&sscreen->shader_parts_mutex);
508 si_destroy_shader_cache(sscreen);
509
510 si_perfcounters_destroy(sscreen);
511 si_gpu_load_kill_thread(sscreen);
512
513 mtx_destroy(&sscreen->gpu_load_mutex);
514 mtx_destroy(&sscreen->aux_context_lock);
515 sscreen->aux_context->destroy(sscreen->aux_context);
516
517 slab_destroy_parent(&sscreen->pool_transfers);
518
519 disk_cache_destroy(sscreen->disk_shader_cache);
520 sscreen->ws->destroy(sscreen->ws);
521 FREE(sscreen);
522 }
523
si_init_gs_info(struct si_screen * sscreen)524 static bool si_init_gs_info(struct si_screen *sscreen)
525 {
526 /* gs_table_depth is not used by GFX9 */
527 if (sscreen->info.chip_class >= GFX9)
528 return true;
529
530 switch (sscreen->info.family) {
531 case CHIP_OLAND:
532 case CHIP_HAINAN:
533 case CHIP_KAVERI:
534 case CHIP_KABINI:
535 case CHIP_MULLINS:
536 case CHIP_ICELAND:
537 case CHIP_CARRIZO:
538 case CHIP_STONEY:
539 sscreen->gs_table_depth = 16;
540 return true;
541 case CHIP_TAHITI:
542 case CHIP_PITCAIRN:
543 case CHIP_VERDE:
544 case CHIP_BONAIRE:
545 case CHIP_HAWAII:
546 case CHIP_TONGA:
547 case CHIP_FIJI:
548 case CHIP_POLARIS10:
549 case CHIP_POLARIS11:
550 case CHIP_POLARIS12:
551 sscreen->gs_table_depth = 32;
552 return true;
553 default:
554 return false;
555 }
556 }
557
si_handle_env_var_force_family(struct si_screen * sscreen)558 static void si_handle_env_var_force_family(struct si_screen *sscreen)
559 {
560 const char *family = debug_get_option("SI_FORCE_FAMILY", NULL);
561 unsigned i;
562
563 if (!family)
564 return;
565
566 for (i = CHIP_TAHITI; i < CHIP_LAST; i++) {
567 if (!strcmp(family, ac_get_llvm_processor_name(i))) {
568 /* Override family and chip_class. */
569 sscreen->info.family = i;
570
571 if (i >= CHIP_VEGA10)
572 sscreen->info.chip_class = GFX9;
573 else if (i >= CHIP_TONGA)
574 sscreen->info.chip_class = VI;
575 else if (i >= CHIP_BONAIRE)
576 sscreen->info.chip_class = CIK;
577 else
578 sscreen->info.chip_class = SI;
579
580 /* Don't submit any IBs. */
581 setenv("RADEON_NOOP", "1", 1);
582 return;
583 }
584 }
585
586 fprintf(stderr, "radeonsi: Unknown family: %s\n", family);
587 exit(1);
588 }
589
si_test_vmfault(struct si_screen * sscreen)590 static void si_test_vmfault(struct si_screen *sscreen)
591 {
592 struct pipe_context *ctx = sscreen->aux_context;
593 struct si_context *sctx = (struct si_context *)ctx;
594 struct pipe_resource *buf =
595 pipe_buffer_create(&sscreen->b, 0, PIPE_USAGE_DEFAULT, 64);
596
597 if (!buf) {
598 puts("Buffer allocation failed.");
599 exit(1);
600 }
601
602 r600_resource(buf)->gpu_address = 0; /* cause a VM fault */
603
604 if (sscreen->debug_flags & DBG(TEST_VMFAULT_CP)) {
605 si_copy_buffer(sctx, buf, buf, 0, 4, 4, 0);
606 ctx->flush(ctx, NULL, 0);
607 puts("VM fault test: CP - done.");
608 }
609 if (sscreen->debug_flags & DBG(TEST_VMFAULT_SDMA)) {
610 sctx->b.dma_clear_buffer(ctx, buf, 0, 4, 0);
611 ctx->flush(ctx, NULL, 0);
612 puts("VM fault test: SDMA - done.");
613 }
614 if (sscreen->debug_flags & DBG(TEST_VMFAULT_SHADER)) {
615 util_test_constant_buffer(ctx, buf);
616 puts("VM fault test: Shader - done.");
617 }
618 exit(0);
619 }
620
si_disk_cache_create(struct si_screen * sscreen)621 static void si_disk_cache_create(struct si_screen *sscreen)
622 {
623 /* Don't use the cache if shader dumping is enabled. */
624 if (sscreen->debug_flags & DBG_ALL_SHADERS)
625 return;
626
627 /* TODO: remove this once gallium supports a nir cache */
628 if (sscreen->debug_flags & DBG(NIR))
629 return;
630
631 uint32_t mesa_timestamp;
632 if (disk_cache_get_function_timestamp(si_disk_cache_create,
633 &mesa_timestamp)) {
634 char *timestamp_str;
635 int res = -1;
636 uint32_t llvm_timestamp;
637
638 if (disk_cache_get_function_timestamp(LLVMInitializeAMDGPUTargetInfo,
639 &llvm_timestamp)) {
640 res = asprintf(×tamp_str, "%u_%u",
641 mesa_timestamp, llvm_timestamp);
642 }
643
644 if (res != -1) {
645 /* These flags affect shader compilation. */
646 uint64_t shader_debug_flags =
647 sscreen->debug_flags &
648 (DBG(FS_CORRECT_DERIVS_AFTER_KILL) |
649 DBG(SI_SCHED) |
650 DBG(UNSAFE_MATH));
651
652 sscreen->disk_shader_cache =
653 disk_cache_create(si_get_family_name(sscreen),
654 timestamp_str,
655 shader_debug_flags);
656 free(timestamp_str);
657 }
658 }
659 }
660
radeonsi_screen_create(struct radeon_winsys * ws,const struct pipe_screen_config * config)661 struct pipe_screen *radeonsi_screen_create(struct radeon_winsys *ws,
662 const struct pipe_screen_config *config)
663 {
664 struct si_screen *sscreen = CALLOC_STRUCT(si_screen);
665 unsigned num_threads, num_compiler_threads, num_compiler_threads_lowprio, i;
666
667 if (!sscreen) {
668 return NULL;
669 }
670
671 sscreen->ws = ws;
672 ws->query_info(ws, &sscreen->info);
673
674 sscreen->debug_flags = debug_get_flags_option("R600_DEBUG",
675 debug_options, 0);
676
677 /* Set functions first. */
678 sscreen->b.context_create = si_pipe_create_context;
679 sscreen->b.destroy = si_destroy_screen;
680
681 si_init_screen_get_functions(sscreen);
682 si_init_screen_buffer_functions(sscreen);
683 si_init_screen_fence_functions(sscreen);
684 si_init_screen_state_functions(sscreen);
685 si_init_screen_texture_functions(sscreen);
686 si_init_screen_query_functions(sscreen);
687
688 /* Set these flags in debug_flags early, so that the shader cache takes
689 * them into account.
690 */
691 if (driQueryOptionb(config->options,
692 "glsl_correct_derivatives_after_discard"))
693 sscreen->debug_flags |= DBG(FS_CORRECT_DERIVS_AFTER_KILL);
694 if (driQueryOptionb(config->options, "radeonsi_enable_sisched"))
695 sscreen->debug_flags |= DBG(SI_SCHED);
696
697
698 if (sscreen->debug_flags & DBG(INFO))
699 ac_print_gpu_info(&sscreen->info);
700
701 slab_create_parent(&sscreen->pool_transfers,
702 sizeof(struct r600_transfer), 64);
703
704 sscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1));
705 if (sscreen->force_aniso >= 0) {
706 printf("radeonsi: Forcing anisotropy filter to %ix\n",
707 /* round down to a power of two */
708 1 << util_logbase2(sscreen->force_aniso));
709 }
710
711 (void) mtx_init(&sscreen->aux_context_lock, mtx_plain);
712 (void) mtx_init(&sscreen->gpu_load_mutex, mtx_plain);
713
714 if (!si_init_gs_info(sscreen) ||
715 !si_init_shader_cache(sscreen)) {
716 FREE(sscreen);
717 return NULL;
718 }
719
720 si_disk_cache_create(sscreen);
721
722 /* Only enable as many threads as we have target machines, but at most
723 * the number of CPUs - 1 if there is more than one.
724 */
725 num_threads = sysconf(_SC_NPROCESSORS_ONLN);
726 num_threads = MAX2(1, num_threads - 1);
727 num_compiler_threads = MIN2(num_threads, ARRAY_SIZE(sscreen->tm));
728 num_compiler_threads_lowprio =
729 MIN2(num_threads, ARRAY_SIZE(sscreen->tm_low_priority));
730
731 if (!util_queue_init(&sscreen->shader_compiler_queue, "si_shader",
732 32, num_compiler_threads,
733 UTIL_QUEUE_INIT_RESIZE_IF_FULL)) {
734 si_destroy_shader_cache(sscreen);
735 FREE(sscreen);
736 return NULL;
737 }
738
739 if (!util_queue_init(&sscreen->shader_compiler_queue_low_priority,
740 "si_shader_low",
741 32, num_compiler_threads_lowprio,
742 UTIL_QUEUE_INIT_RESIZE_IF_FULL |
743 UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY)) {
744 si_destroy_shader_cache(sscreen);
745 FREE(sscreen);
746 return NULL;
747 }
748
749 si_handle_env_var_force_family(sscreen);
750
751 if (!debug_get_bool_option("RADEON_DISABLE_PERFCOUNTERS", false))
752 si_init_perfcounters(sscreen);
753
754 /* Hawaii has a bug with offchip buffers > 256 that can be worked
755 * around by setting 4K granularity.
756 */
757 sscreen->tess_offchip_block_dw_size =
758 sscreen->info.family == CHIP_HAWAII ? 4096 : 8192;
759
760 /* The mere presense of CLEAR_STATE in the IB causes random GPU hangs
761 * on SI. */
762 sscreen->has_clear_state = sscreen->info.chip_class >= CIK;
763
764 sscreen->has_distributed_tess =
765 sscreen->info.chip_class >= VI &&
766 sscreen->info.max_se >= 2;
767
768 sscreen->has_draw_indirect_multi =
769 (sscreen->info.family >= CHIP_POLARIS10) ||
770 (sscreen->info.chip_class == VI &&
771 sscreen->info.pfp_fw_version >= 121 &&
772 sscreen->info.me_fw_version >= 87) ||
773 (sscreen->info.chip_class == CIK &&
774 sscreen->info.pfp_fw_version >= 211 &&
775 sscreen->info.me_fw_version >= 173) ||
776 (sscreen->info.chip_class == SI &&
777 sscreen->info.pfp_fw_version >= 79 &&
778 sscreen->info.me_fw_version >= 142);
779
780 sscreen->has_out_of_order_rast = sscreen->info.chip_class >= VI &&
781 sscreen->info.max_se >= 2 &&
782 !(sscreen->debug_flags & DBG(NO_OUT_OF_ORDER));
783 sscreen->assume_no_z_fights =
784 driQueryOptionb(config->options, "radeonsi_assume_no_z_fights");
785 sscreen->commutative_blend_add =
786 driQueryOptionb(config->options, "radeonsi_commutative_blend_add");
787 sscreen->clear_db_cache_before_clear =
788 driQueryOptionb(config->options, "radeonsi_clear_db_cache_before_clear");
789 sscreen->has_msaa_sample_loc_bug = (sscreen->info.family >= CHIP_POLARIS10 &&
790 sscreen->info.family <= CHIP_POLARIS12) ||
791 sscreen->info.family == CHIP_VEGA10 ||
792 sscreen->info.family == CHIP_RAVEN;
793 sscreen->has_ls_vgpr_init_bug = sscreen->info.family == CHIP_VEGA10 ||
794 sscreen->info.family == CHIP_RAVEN;
795
796 if (sscreen->debug_flags & DBG(DPBB)) {
797 sscreen->dpbb_allowed = true;
798 } else {
799 /* Only enable primitive binning on Raven by default. */
800 sscreen->dpbb_allowed = sscreen->info.family == CHIP_RAVEN &&
801 !(sscreen->debug_flags & DBG(NO_DPBB));
802 }
803
804 if (sscreen->debug_flags & DBG(DFSM)) {
805 sscreen->dfsm_allowed = sscreen->dpbb_allowed;
806 } else {
807 sscreen->dfsm_allowed = sscreen->dpbb_allowed &&
808 !(sscreen->debug_flags & DBG(NO_DFSM));
809 }
810
811 /* While it would be nice not to have this flag, we are constrained
812 * by the reality that LLVM 5.0 doesn't have working VGPR indexing
813 * on GFX9.
814 */
815 sscreen->llvm_has_working_vgpr_indexing = sscreen->info.chip_class <= VI;
816
817 /* Some chips have RB+ registers, but don't support RB+. Those must
818 * always disable it.
819 */
820 if (sscreen->info.family == CHIP_STONEY ||
821 sscreen->info.chip_class >= GFX9) {
822 sscreen->has_rbplus = true;
823
824 sscreen->rbplus_allowed =
825 !(sscreen->debug_flags & DBG(NO_RB_PLUS)) &&
826 (sscreen->info.family == CHIP_STONEY ||
827 sscreen->info.family == CHIP_RAVEN);
828 }
829
830 sscreen->dcc_msaa_allowed =
831 !(sscreen->debug_flags & DBG(NO_DCC_MSAA)) &&
832 (sscreen->debug_flags & DBG(DCC_MSAA) ||
833 sscreen->info.chip_class == VI);
834
835 sscreen->cpdma_prefetch_writes_memory = sscreen->info.chip_class <= VI;
836
837 (void) mtx_init(&sscreen->shader_parts_mutex, mtx_plain);
838 sscreen->use_monolithic_shaders =
839 (sscreen->debug_flags & DBG(MONOLITHIC_SHADERS)) != 0;
840
841 sscreen->barrier_flags.cp_to_L2 = SI_CONTEXT_INV_SMEM_L1 |
842 SI_CONTEXT_INV_VMEM_L1;
843 if (sscreen->info.chip_class <= VI) {
844 sscreen->barrier_flags.cp_to_L2 |= SI_CONTEXT_INV_GLOBAL_L2;
845 sscreen->barrier_flags.L2_to_cp |= SI_CONTEXT_WRITEBACK_GLOBAL_L2;
846 }
847
848 if (debug_get_bool_option("RADEON_DUMP_SHADERS", false))
849 sscreen->debug_flags |= DBG_ALL_SHADERS;
850
851 for (i = 0; i < num_compiler_threads; i++)
852 sscreen->tm[i] = si_create_llvm_target_machine(sscreen);
853 for (i = 0; i < num_compiler_threads_lowprio; i++)
854 sscreen->tm_low_priority[i] = si_create_llvm_target_machine(sscreen);
855
856 /* Create the auxiliary context. This must be done last. */
857 sscreen->aux_context = si_create_context(&sscreen->b, 0);
858
859 if (sscreen->debug_flags & DBG(TEST_DMA))
860 si_test_dma(sscreen);
861
862 if (sscreen->debug_flags & (DBG(TEST_VMFAULT_CP) |
863 DBG(TEST_VMFAULT_SDMA) |
864 DBG(TEST_VMFAULT_SHADER)))
865 si_test_vmfault(sscreen);
866
867 return &sscreen->b;
868 }
869