• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * on the rights to use, copy, modify, merge, publish, distribute, sub
8  * license, and/or sell copies of the Software, and to permit persons to whom
9  * the Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "si_pipe.h"
25 #include "si_public.h"
26 #include "si_shader_internal.h"
27 #include "sid.h"
28 
29 #include "radeon/r600_cs.h"
30 #include "radeon/radeon_uvd.h"
31 #include "util/hash_table.h"
32 #include "util/u_log.h"
33 #include "util/u_memory.h"
34 #include "util/u_suballoc.h"
35 #include "util/u_tests.h"
36 #include "util/xmlconfig.h"
37 #include "vl/vl_decoder.h"
38 #include "../ddebug/dd_util.h"
39 
40 static const struct debug_named_value debug_options[] = {
41 	/* Shader logging options: */
42 	{ "vs", DBG(VS), "Print vertex shaders" },
43 	{ "ps", DBG(PS), "Print pixel shaders" },
44 	{ "gs", DBG(GS), "Print geometry shaders" },
45 	{ "tcs", DBG(TCS), "Print tessellation control shaders" },
46 	{ "tes", DBG(TES), "Print tessellation evaluation shaders" },
47 	{ "cs", DBG(CS), "Print compute shaders" },
48 	{ "noir", DBG(NO_IR), "Don't print the LLVM IR"},
49 	{ "notgsi", DBG(NO_TGSI), "Don't print the TGSI"},
50 	{ "noasm", DBG(NO_ASM), "Don't print disassembled shaders"},
51 	{ "preoptir", DBG(PREOPT_IR), "Print the LLVM IR before initial optimizations" },
52 
53 	/* Shader compiler options the shader cache should be aware of: */
54 	{ "unsafemath", DBG(UNSAFE_MATH), "Enable unsafe math shader optimizations" },
55 	{ "sisched", DBG(SI_SCHED), "Enable LLVM SI Machine Instruction Scheduler." },
56 
57 	/* Shader compiler options (with no effect on the shader cache): */
58 	{ "checkir", DBG(CHECK_IR), "Enable additional sanity checks on shader IR" },
59 	{ "precompile", DBG(PRECOMPILE), "Compile one shader variant at shader creation." },
60 	{ "nir", DBG(NIR), "Enable experimental NIR shaders" },
61 	{ "mono", DBG(MONOLITHIC_SHADERS), "Use old-style monolithic shaders compiled on demand" },
62 	{ "nooptvariant", DBG(NO_OPT_VARIANT), "Disable compiling optimized shader variants." },
63 
64 	/* Information logging options: */
65 	{ "info", DBG(INFO), "Print driver information" },
66 	{ "tex", DBG(TEX), "Print texture info" },
67 	{ "compute", DBG(COMPUTE), "Print compute info" },
68 	{ "vm", DBG(VM), "Print virtual addresses when creating resources" },
69 
70 	/* Driver options: */
71 	{ "forcedma", DBG(FORCE_DMA), "Use asynchronous DMA for all operations when possible." },
72 	{ "nodma", DBG(NO_ASYNC_DMA), "Disable asynchronous DMA" },
73 	{ "nowc", DBG(NO_WC), "Disable GTT write combining" },
74 	{ "check_vm", DBG(CHECK_VM), "Check VM faults and dump debug info." },
75 	{ "reserve_vmid", DBG(RESERVE_VMID), "Force VMID reservation per context." },
76 
77 	/* 3D engine options: */
78 	{ "switch_on_eop", DBG(SWITCH_ON_EOP), "Program WD/IA to switch on end-of-packet." },
79 	{ "nooutoforder", DBG(NO_OUT_OF_ORDER), "Disable out-of-order rasterization" },
80 	{ "nodpbb", DBG(NO_DPBB), "Disable DPBB." },
81 	{ "nodfsm", DBG(NO_DFSM), "Disable DFSM." },
82 	{ "dpbb", DBG(DPBB), "Enable DPBB." },
83 	{ "dfsm", DBG(DFSM), "Enable DFSM." },
84 	{ "nohyperz", DBG(NO_HYPERZ), "Disable Hyper-Z" },
85 	{ "norbplus", DBG(NO_RB_PLUS), "Disable RB+." },
86 	{ "no2d", DBG(NO_2D_TILING), "Disable 2D tiling" },
87 	{ "notiling", DBG(NO_TILING), "Disable tiling" },
88 	{ "nodcc", DBG(NO_DCC), "Disable DCC." },
89 	{ "nodccclear", DBG(NO_DCC_CLEAR), "Disable DCC fast clear." },
90 	{ "nodccfb", DBG(NO_DCC_FB), "Disable separate DCC on the main framebuffer" },
91 	{ "nodccmsaa", DBG(NO_DCC_MSAA), "Disable DCC for MSAA" },
92 	{ "dccmsaa", DBG(DCC_MSAA), "Enable DCC for MSAA" },
93 
94 	/* Tests: */
95 	{ "testdma", DBG(TEST_DMA), "Invoke SDMA tests and exit." },
96 	{ "testvmfaultcp", DBG(TEST_VMFAULT_CP), "Invoke a CP VM fault test and exit." },
97 	{ "testvmfaultsdma", DBG(TEST_VMFAULT_SDMA), "Invoke a SDMA VM fault test and exit." },
98 	{ "testvmfaultshader", DBG(TEST_VMFAULT_SHADER), "Invoke a shader VM fault test and exit." },
99 
100 	DEBUG_NAMED_VALUE_END /* must be last */
101 };
102 
103 /*
104  * pipe_context
105  */
si_destroy_context(struct pipe_context * context)106 static void si_destroy_context(struct pipe_context *context)
107 {
108 	struct si_context *sctx = (struct si_context *)context;
109 	int i;
110 
111 	/* Unreference the framebuffer normally to disable related logic
112 	 * properly.
113 	 */
114 	struct pipe_framebuffer_state fb = {};
115 	if (context->set_framebuffer_state)
116 		context->set_framebuffer_state(context, &fb);
117 
118 	si_release_all_descriptors(sctx);
119 
120 	pipe_resource_reference(&sctx->esgs_ring, NULL);
121 	pipe_resource_reference(&sctx->gsvs_ring, NULL);
122 	pipe_resource_reference(&sctx->tf_ring, NULL);
123 	pipe_resource_reference(&sctx->tess_offchip_ring, NULL);
124 	pipe_resource_reference(&sctx->null_const_buf.buffer, NULL);
125 	r600_resource_reference(&sctx->border_color_buffer, NULL);
126 	free(sctx->border_color_table);
127 	r600_resource_reference(&sctx->scratch_buffer, NULL);
128 	r600_resource_reference(&sctx->compute_scratch_buffer, NULL);
129 	r600_resource_reference(&sctx->wait_mem_scratch, NULL);
130 
131 	si_pm4_free_state(sctx, sctx->init_config, ~0);
132 	if (sctx->init_config_gs_rings)
133 		si_pm4_free_state(sctx, sctx->init_config_gs_rings, ~0);
134 	for (i = 0; i < ARRAY_SIZE(sctx->vgt_shader_config); i++)
135 		si_pm4_delete_state(sctx, vgt_shader_config, sctx->vgt_shader_config[i]);
136 
137 	if (sctx->fixed_func_tcs_shader.cso)
138 		sctx->b.b.delete_tcs_state(&sctx->b.b, sctx->fixed_func_tcs_shader.cso);
139 	if (sctx->custom_dsa_flush)
140 		sctx->b.b.delete_depth_stencil_alpha_state(&sctx->b.b, sctx->custom_dsa_flush);
141 	if (sctx->custom_blend_resolve)
142 		sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_resolve);
143 	if (sctx->custom_blend_fmask_decompress)
144 		sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_fmask_decompress);
145 	if (sctx->custom_blend_eliminate_fastclear)
146 		sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_eliminate_fastclear);
147 	if (sctx->custom_blend_dcc_decompress)
148 		sctx->b.b.delete_blend_state(&sctx->b.b, sctx->custom_blend_dcc_decompress);
149 	if (sctx->vs_blit_pos)
150 		sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_pos);
151 	if (sctx->vs_blit_pos_layered)
152 		sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_pos_layered);
153 	if (sctx->vs_blit_color)
154 		sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_color);
155 	if (sctx->vs_blit_color_layered)
156 		sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_color_layered);
157 	if (sctx->vs_blit_texcoord)
158 		sctx->b.b.delete_vs_state(&sctx->b.b, sctx->vs_blit_texcoord);
159 
160 	if (sctx->blitter)
161 		util_blitter_destroy(sctx->blitter);
162 
163 	si_common_context_cleanup(&sctx->b);
164 
165 	LLVMDisposeTargetMachine(sctx->tm);
166 
167 	si_saved_cs_reference(&sctx->current_saved_cs, NULL);
168 
169 	_mesa_hash_table_destroy(sctx->tex_handles, NULL);
170 	_mesa_hash_table_destroy(sctx->img_handles, NULL);
171 
172 	util_dynarray_fini(&sctx->resident_tex_handles);
173 	util_dynarray_fini(&sctx->resident_img_handles);
174 	util_dynarray_fini(&sctx->resident_tex_needs_color_decompress);
175 	util_dynarray_fini(&sctx->resident_img_needs_color_decompress);
176 	util_dynarray_fini(&sctx->resident_tex_needs_depth_decompress);
177 	FREE(sctx);
178 }
179 
180 static enum pipe_reset_status
si_amdgpu_get_reset_status(struct pipe_context * ctx)181 si_amdgpu_get_reset_status(struct pipe_context *ctx)
182 {
183 	struct si_context *sctx = (struct si_context *)ctx;
184 
185 	return sctx->b.ws->ctx_query_reset_status(sctx->b.ctx);
186 }
187 
188 /* Apitrace profiling:
189  *   1) qapitrace : Tools -> Profile: Measure CPU & GPU times
190  *   2) In the middle panel, zoom in (mouse wheel) on some bad draw call
191  *      and remember its number.
192  *   3) In Mesa, enable queries and performance counters around that draw
193  *      call and print the results.
194  *   4) glretrace --benchmark --markers ..
195  */
si_emit_string_marker(struct pipe_context * ctx,const char * string,int len)196 static void si_emit_string_marker(struct pipe_context *ctx,
197 				  const char *string, int len)
198 {
199 	struct si_context *sctx = (struct si_context *)ctx;
200 
201 	dd_parse_apitrace_marker(string, len, &sctx->apitrace_call_number);
202 
203 	if (sctx->b.log)
204 		u_log_printf(sctx->b.log, "\nString marker: %*s\n", len, string);
205 }
206 
207 static LLVMTargetMachineRef
si_create_llvm_target_machine(struct si_screen * sscreen)208 si_create_llvm_target_machine(struct si_screen *sscreen)
209 {
210 	enum ac_target_machine_options tm_options =
211 		(sscreen->debug_flags & DBG(SI_SCHED) ? AC_TM_SISCHED : 0) |
212 		(sscreen->info.chip_class >= GFX9 ? AC_TM_FORCE_ENABLE_XNACK : 0) |
213 		(sscreen->info.chip_class < GFX9 ? AC_TM_FORCE_DISABLE_XNACK : 0) |
214 		(!sscreen->llvm_has_working_vgpr_indexing ? AC_TM_PROMOTE_ALLOCA_TO_SCRATCH : 0);
215 
216 	return ac_create_target_machine(sscreen->info.family, tm_options);
217 }
218 
si_set_debug_callback(struct pipe_context * ctx,const struct pipe_debug_callback * cb)219 static void si_set_debug_callback(struct pipe_context *ctx,
220 				  const struct pipe_debug_callback *cb)
221 {
222 	struct si_context *sctx = (struct si_context *)ctx;
223 	struct si_screen *screen = sctx->screen;
224 
225 	util_queue_finish(&screen->shader_compiler_queue);
226 	util_queue_finish(&screen->shader_compiler_queue_low_priority);
227 
228 	if (cb)
229 		sctx->debug = *cb;
230 	else
231 		memset(&sctx->debug, 0, sizeof(sctx->debug));
232 }
233 
si_set_log_context(struct pipe_context * ctx,struct u_log_context * log)234 static void si_set_log_context(struct pipe_context *ctx,
235 			       struct u_log_context *log)
236 {
237 	struct si_context *sctx = (struct si_context *)ctx;
238 	sctx->b.log = log;
239 
240 	if (log)
241 		u_log_add_auto_logger(log, si_auto_log_cs, sctx);
242 }
243 
si_create_context(struct pipe_screen * screen,unsigned flags)244 static struct pipe_context *si_create_context(struct pipe_screen *screen,
245                                               unsigned flags)
246 {
247 	struct si_context *sctx = CALLOC_STRUCT(si_context);
248 	struct si_screen* sscreen = (struct si_screen *)screen;
249 	struct radeon_winsys *ws = sscreen->ws;
250 	int shader, i;
251 
252 	if (!sctx)
253 		return NULL;
254 
255 	if (flags & PIPE_CONTEXT_DEBUG)
256 		sscreen->record_llvm_ir = true; /* racy but not critical */
257 
258 	sctx->b.b.screen = screen; /* this must be set first */
259 	sctx->b.b.priv = NULL;
260 	sctx->b.b.destroy = si_destroy_context;
261 	sctx->b.b.emit_string_marker = si_emit_string_marker;
262 	sctx->b.b.set_debug_callback = si_set_debug_callback;
263 	sctx->b.b.set_log_context = si_set_log_context;
264 	sctx->b.set_atom_dirty = (void *)si_set_atom_dirty;
265 	sctx->screen = sscreen; /* Easy accessing of screen/winsys. */
266 	sctx->is_debug = (flags & PIPE_CONTEXT_DEBUG) != 0;
267 
268 	if (!si_common_context_init(&sctx->b, sscreen, flags))
269 		goto fail;
270 
271 	if (sscreen->info.drm_major == 3)
272 		sctx->b.b.get_device_reset_status = si_amdgpu_get_reset_status;
273 
274 	si_init_buffer_functions(sctx);
275 	si_init_clear_functions(sctx);
276 	si_init_blit_functions(sctx);
277 	si_init_compute_functions(sctx);
278 	si_init_cp_dma_functions(sctx);
279 	si_init_debug_functions(sctx);
280 	si_init_msaa_functions(sctx);
281 	si_init_streamout_functions(sctx);
282 
283 	if (sscreen->info.has_hw_decode) {
284 		sctx->b.b.create_video_codec = si_uvd_create_decoder;
285 		sctx->b.b.create_video_buffer = si_video_buffer_create;
286 	} else {
287 		sctx->b.b.create_video_codec = vl_create_decoder;
288 		sctx->b.b.create_video_buffer = vl_video_buffer_create;
289 	}
290 
291 	sctx->b.gfx.cs = ws->cs_create(sctx->b.ctx, RING_GFX,
292 				       si_context_gfx_flush, sctx);
293 	sctx->b.gfx.flush = si_context_gfx_flush;
294 
295 	/* Border colors. */
296 	sctx->border_color_table = malloc(SI_MAX_BORDER_COLORS *
297 					  sizeof(*sctx->border_color_table));
298 	if (!sctx->border_color_table)
299 		goto fail;
300 
301 	sctx->border_color_buffer = (struct r600_resource*)
302 		pipe_buffer_create(screen, 0, PIPE_USAGE_DEFAULT,
303 				   SI_MAX_BORDER_COLORS *
304 				   sizeof(*sctx->border_color_table));
305 	if (!sctx->border_color_buffer)
306 		goto fail;
307 
308 	sctx->border_color_map =
309 		ws->buffer_map(sctx->border_color_buffer->buf,
310 			       NULL, PIPE_TRANSFER_WRITE);
311 	if (!sctx->border_color_map)
312 		goto fail;
313 
314 	si_init_all_descriptors(sctx);
315 	si_init_fence_functions(sctx);
316 	si_init_state_functions(sctx);
317 	si_init_shader_functions(sctx);
318 	si_init_viewport_functions(sctx);
319 	si_init_ia_multi_vgt_param_table(sctx);
320 
321 	if (sctx->b.chip_class >= CIK)
322 		cik_init_sdma_functions(sctx);
323 	else
324 		si_init_dma_functions(sctx);
325 
326 	if (sscreen->debug_flags & DBG(FORCE_DMA))
327 		sctx->b.b.resource_copy_region = sctx->b.dma_copy;
328 
329 	sctx->blitter = util_blitter_create(&sctx->b.b);
330 	if (sctx->blitter == NULL)
331 		goto fail;
332 	sctx->blitter->draw_rectangle = si_draw_rectangle;
333 	sctx->blitter->skip_viewport_restore = true;
334 
335 	sctx->sample_mask.sample_mask = 0xffff;
336 
337 	if (sctx->b.chip_class >= GFX9) {
338 		sctx->wait_mem_scratch = (struct r600_resource*)
339 			pipe_buffer_create(screen, 0, PIPE_USAGE_DEFAULT, 4);
340 		if (!sctx->wait_mem_scratch)
341 			goto fail;
342 
343 		/* Initialize the memory. */
344 		struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
345 		radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
346 		radeon_emit(cs, S_370_DST_SEL(V_370_MEMORY_SYNC) |
347 			    S_370_WR_CONFIRM(1) |
348 			    S_370_ENGINE_SEL(V_370_ME));
349 		radeon_emit(cs, sctx->wait_mem_scratch->gpu_address);
350 		radeon_emit(cs, sctx->wait_mem_scratch->gpu_address >> 32);
351 		radeon_emit(cs, sctx->wait_mem_number);
352 		radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
353 					  sctx->wait_mem_scratch,
354 					  RADEON_USAGE_WRITE, RADEON_PRIO_FENCE);
355 	}
356 
357 	/* CIK cannot unbind a constant buffer (S_BUFFER_LOAD doesn't skip loads
358 	 * if NUM_RECORDS == 0). We need to use a dummy buffer instead. */
359 	if (sctx->b.chip_class == CIK) {
360 		sctx->null_const_buf.buffer =
361 			si_aligned_buffer_create(screen,
362 						   R600_RESOURCE_FLAG_UNMAPPABLE,
363 						   PIPE_USAGE_DEFAULT, 16,
364 						   sctx->screen->info.tcc_cache_line_size);
365 		if (!sctx->null_const_buf.buffer)
366 			goto fail;
367 		sctx->null_const_buf.buffer_size = sctx->null_const_buf.buffer->width0;
368 
369 		for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
370 			for (i = 0; i < SI_NUM_CONST_BUFFERS; i++) {
371 				sctx->b.b.set_constant_buffer(&sctx->b.b, shader, i,
372 							      &sctx->null_const_buf);
373 			}
374 		}
375 
376 		si_set_rw_buffer(sctx, SI_HS_CONST_DEFAULT_TESS_LEVELS,
377 				 &sctx->null_const_buf);
378 		si_set_rw_buffer(sctx, SI_VS_CONST_INSTANCE_DIVISORS,
379 				 &sctx->null_const_buf);
380 		si_set_rw_buffer(sctx, SI_VS_CONST_CLIP_PLANES,
381 				 &sctx->null_const_buf);
382 		si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE,
383 				 &sctx->null_const_buf);
384 		si_set_rw_buffer(sctx, SI_PS_CONST_SAMPLE_POSITIONS,
385 				 &sctx->null_const_buf);
386 
387 		/* Clear the NULL constant buffer, because loads should return zeros. */
388 		si_clear_buffer(&sctx->b.b, sctx->null_const_buf.buffer, 0,
389 				sctx->null_const_buf.buffer->width0, 0,
390 				R600_COHERENCY_SHADER);
391 	}
392 
393 	uint64_t max_threads_per_block;
394 	screen->get_compute_param(screen, PIPE_SHADER_IR_TGSI,
395 				  PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK,
396 				  &max_threads_per_block);
397 
398 	/* The maximum number of scratch waves. Scratch space isn't divided
399 	 * evenly between CUs. The number is only a function of the number of CUs.
400 	 * We can decrease the constant to decrease the scratch buffer size.
401 	 *
402 	 * sctx->scratch_waves must be >= the maximum posible size of
403 	 * 1 threadgroup, so that the hw doesn't hang from being unable
404 	 * to start any.
405 	 *
406 	 * The recommended value is 4 per CU at most. Higher numbers don't
407 	 * bring much benefit, but they still occupy chip resources (think
408 	 * async compute). I've seen ~2% performance difference between 4 and 32.
409 	 */
410 	sctx->scratch_waves = MAX2(32 * sscreen->info.num_good_compute_units,
411 				   max_threads_per_block / 64);
412 
413 	sctx->tm = si_create_llvm_target_machine(sscreen);
414 
415 	/* Bindless handles. */
416 	sctx->tex_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
417 						    _mesa_key_pointer_equal);
418 	sctx->img_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
419 						    _mesa_key_pointer_equal);
420 
421 	util_dynarray_init(&sctx->resident_tex_handles, NULL);
422 	util_dynarray_init(&sctx->resident_img_handles, NULL);
423 	util_dynarray_init(&sctx->resident_tex_needs_color_decompress, NULL);
424 	util_dynarray_init(&sctx->resident_img_needs_color_decompress, NULL);
425 	util_dynarray_init(&sctx->resident_tex_needs_depth_decompress, NULL);
426 
427 	/* this must be last */
428 	si_begin_new_cs(sctx);
429 	return &sctx->b.b;
430 fail:
431 	fprintf(stderr, "radeonsi: Failed to create a context.\n");
432 	si_destroy_context(&sctx->b.b);
433 	return NULL;
434 }
435 
si_pipe_create_context(struct pipe_screen * screen,void * priv,unsigned flags)436 static struct pipe_context *si_pipe_create_context(struct pipe_screen *screen,
437 						   void *priv, unsigned flags)
438 {
439 	struct si_screen *sscreen = (struct si_screen *)screen;
440 	struct pipe_context *ctx;
441 
442 	if (sscreen->debug_flags & DBG(CHECK_VM))
443 		flags |= PIPE_CONTEXT_DEBUG;
444 
445 	ctx = si_create_context(screen, flags);
446 
447 	if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
448 		return ctx;
449 
450 	/* Clover (compute-only) is unsupported. */
451 	if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
452 		return ctx;
453 
454 	/* When shaders are logged to stderr, asynchronous compilation is
455 	 * disabled too. */
456 	if (sscreen->debug_flags & DBG_ALL_SHADERS)
457 		return ctx;
458 
459 	/* Use asynchronous flushes only on amdgpu, since the radeon
460 	 * implementation for fence_server_sync is incomplete. */
461 	return threaded_context_create(ctx, &sscreen->pool_transfers,
462 				       si_replace_buffer_storage,
463 				       sscreen->info.drm_major >= 3 ? si_create_fence : NULL,
464 				       &((struct si_context*)ctx)->b.tc);
465 }
466 
467 /*
468  * pipe_screen
469  */
470 
si_destroy_screen(struct pipe_screen * pscreen)471 static void si_destroy_screen(struct pipe_screen* pscreen)
472 {
473 	struct si_screen *sscreen = (struct si_screen *)pscreen;
474 	struct si_shader_part *parts[] = {
475 		sscreen->vs_prologs,
476 		sscreen->tcs_epilogs,
477 		sscreen->gs_prologs,
478 		sscreen->ps_prologs,
479 		sscreen->ps_epilogs
480 	};
481 	unsigned i;
482 
483 	if (!sscreen->ws->unref(sscreen->ws))
484 		return;
485 
486 	util_queue_destroy(&sscreen->shader_compiler_queue);
487 	util_queue_destroy(&sscreen->shader_compiler_queue_low_priority);
488 
489 	for (i = 0; i < ARRAY_SIZE(sscreen->tm); i++)
490 		if (sscreen->tm[i])
491 			LLVMDisposeTargetMachine(sscreen->tm[i]);
492 
493 	for (i = 0; i < ARRAY_SIZE(sscreen->tm_low_priority); i++)
494 		if (sscreen->tm_low_priority[i])
495 			LLVMDisposeTargetMachine(sscreen->tm_low_priority[i]);
496 
497 	/* Free shader parts. */
498 	for (i = 0; i < ARRAY_SIZE(parts); i++) {
499 		while (parts[i]) {
500 			struct si_shader_part *part = parts[i];
501 
502 			parts[i] = part->next;
503 			ac_shader_binary_clean(&part->binary);
504 			FREE(part);
505 		}
506 	}
507 	mtx_destroy(&sscreen->shader_parts_mutex);
508 	si_destroy_shader_cache(sscreen);
509 
510 	si_perfcounters_destroy(sscreen);
511 	si_gpu_load_kill_thread(sscreen);
512 
513 	mtx_destroy(&sscreen->gpu_load_mutex);
514 	mtx_destroy(&sscreen->aux_context_lock);
515 	sscreen->aux_context->destroy(sscreen->aux_context);
516 
517 	slab_destroy_parent(&sscreen->pool_transfers);
518 
519 	disk_cache_destroy(sscreen->disk_shader_cache);
520 	sscreen->ws->destroy(sscreen->ws);
521 	FREE(sscreen);
522 }
523 
si_init_gs_info(struct si_screen * sscreen)524 static bool si_init_gs_info(struct si_screen *sscreen)
525 {
526 	/* gs_table_depth is not used by GFX9 */
527 	if (sscreen->info.chip_class >= GFX9)
528 		return true;
529 
530 	switch (sscreen->info.family) {
531 	case CHIP_OLAND:
532 	case CHIP_HAINAN:
533 	case CHIP_KAVERI:
534 	case CHIP_KABINI:
535 	case CHIP_MULLINS:
536 	case CHIP_ICELAND:
537 	case CHIP_CARRIZO:
538 	case CHIP_STONEY:
539 		sscreen->gs_table_depth = 16;
540 		return true;
541 	case CHIP_TAHITI:
542 	case CHIP_PITCAIRN:
543 	case CHIP_VERDE:
544 	case CHIP_BONAIRE:
545 	case CHIP_HAWAII:
546 	case CHIP_TONGA:
547 	case CHIP_FIJI:
548 	case CHIP_POLARIS10:
549 	case CHIP_POLARIS11:
550 	case CHIP_POLARIS12:
551 		sscreen->gs_table_depth = 32;
552 		return true;
553 	default:
554 		return false;
555 	}
556 }
557 
si_handle_env_var_force_family(struct si_screen * sscreen)558 static void si_handle_env_var_force_family(struct si_screen *sscreen)
559 {
560 	const char *family = debug_get_option("SI_FORCE_FAMILY", NULL);
561 	unsigned i;
562 
563 	if (!family)
564 		return;
565 
566 	for (i = CHIP_TAHITI; i < CHIP_LAST; i++) {
567 		if (!strcmp(family, ac_get_llvm_processor_name(i))) {
568 			/* Override family and chip_class. */
569 			sscreen->info.family = i;
570 
571 			if (i >= CHIP_VEGA10)
572 				sscreen->info.chip_class = GFX9;
573 			else if (i >= CHIP_TONGA)
574 				sscreen->info.chip_class = VI;
575 			else if (i >= CHIP_BONAIRE)
576 				sscreen->info.chip_class = CIK;
577 			else
578 				sscreen->info.chip_class = SI;
579 
580 			/* Don't submit any IBs. */
581 			setenv("RADEON_NOOP", "1", 1);
582 			return;
583 		}
584 	}
585 
586 	fprintf(stderr, "radeonsi: Unknown family: %s\n", family);
587 	exit(1);
588 }
589 
si_test_vmfault(struct si_screen * sscreen)590 static void si_test_vmfault(struct si_screen *sscreen)
591 {
592 	struct pipe_context *ctx = sscreen->aux_context;
593 	struct si_context *sctx = (struct si_context *)ctx;
594 	struct pipe_resource *buf =
595 		pipe_buffer_create(&sscreen->b, 0, PIPE_USAGE_DEFAULT, 64);
596 
597 	if (!buf) {
598 		puts("Buffer allocation failed.");
599 		exit(1);
600 	}
601 
602 	r600_resource(buf)->gpu_address = 0; /* cause a VM fault */
603 
604 	if (sscreen->debug_flags & DBG(TEST_VMFAULT_CP)) {
605 		si_copy_buffer(sctx, buf, buf, 0, 4, 4, 0);
606 		ctx->flush(ctx, NULL, 0);
607 		puts("VM fault test: CP - done.");
608 	}
609 	if (sscreen->debug_flags & DBG(TEST_VMFAULT_SDMA)) {
610 		sctx->b.dma_clear_buffer(ctx, buf, 0, 4, 0);
611 		ctx->flush(ctx, NULL, 0);
612 		puts("VM fault test: SDMA - done.");
613 	}
614 	if (sscreen->debug_flags & DBG(TEST_VMFAULT_SHADER)) {
615 		util_test_constant_buffer(ctx, buf);
616 		puts("VM fault test: Shader - done.");
617 	}
618 	exit(0);
619 }
620 
si_disk_cache_create(struct si_screen * sscreen)621 static void si_disk_cache_create(struct si_screen *sscreen)
622 {
623 	/* Don't use the cache if shader dumping is enabled. */
624 	if (sscreen->debug_flags & DBG_ALL_SHADERS)
625 		return;
626 
627 	/* TODO: remove this once gallium supports a nir cache */
628 	if (sscreen->debug_flags & DBG(NIR))
629 		return;
630 
631 	uint32_t mesa_timestamp;
632 	if (disk_cache_get_function_timestamp(si_disk_cache_create,
633 					      &mesa_timestamp)) {
634 		char *timestamp_str;
635 		int res = -1;
636 		uint32_t llvm_timestamp;
637 
638 		if (disk_cache_get_function_timestamp(LLVMInitializeAMDGPUTargetInfo,
639 						      &llvm_timestamp)) {
640 			res = asprintf(&timestamp_str, "%u_%u",
641 				       mesa_timestamp, llvm_timestamp);
642 		}
643 
644 		if (res != -1) {
645 			/* These flags affect shader compilation. */
646 			uint64_t shader_debug_flags =
647 				sscreen->debug_flags &
648 				(DBG(FS_CORRECT_DERIVS_AFTER_KILL) |
649 				 DBG(SI_SCHED) |
650 				 DBG(UNSAFE_MATH));
651 
652 			sscreen->disk_shader_cache =
653 				disk_cache_create(si_get_family_name(sscreen),
654 						  timestamp_str,
655 						  shader_debug_flags);
656 			free(timestamp_str);
657 		}
658 	}
659 }
660 
radeonsi_screen_create(struct radeon_winsys * ws,const struct pipe_screen_config * config)661 struct pipe_screen *radeonsi_screen_create(struct radeon_winsys *ws,
662 					   const struct pipe_screen_config *config)
663 {
664 	struct si_screen *sscreen = CALLOC_STRUCT(si_screen);
665 	unsigned num_threads, num_compiler_threads, num_compiler_threads_lowprio, i;
666 
667 	if (!sscreen) {
668 		return NULL;
669 	}
670 
671 	sscreen->ws = ws;
672 	ws->query_info(ws, &sscreen->info);
673 
674 	sscreen->debug_flags = debug_get_flags_option("R600_DEBUG",
675 							debug_options, 0);
676 
677 	/* Set functions first. */
678 	sscreen->b.context_create = si_pipe_create_context;
679 	sscreen->b.destroy = si_destroy_screen;
680 
681 	si_init_screen_get_functions(sscreen);
682 	si_init_screen_buffer_functions(sscreen);
683 	si_init_screen_fence_functions(sscreen);
684 	si_init_screen_state_functions(sscreen);
685 	si_init_screen_texture_functions(sscreen);
686 	si_init_screen_query_functions(sscreen);
687 
688 	/* Set these flags in debug_flags early, so that the shader cache takes
689 	 * them into account.
690 	 */
691 	if (driQueryOptionb(config->options,
692 			    "glsl_correct_derivatives_after_discard"))
693 		sscreen->debug_flags |= DBG(FS_CORRECT_DERIVS_AFTER_KILL);
694 	if (driQueryOptionb(config->options, "radeonsi_enable_sisched"))
695 		sscreen->debug_flags |= DBG(SI_SCHED);
696 
697 
698 	if (sscreen->debug_flags & DBG(INFO))
699 		ac_print_gpu_info(&sscreen->info);
700 
701 	slab_create_parent(&sscreen->pool_transfers,
702 			   sizeof(struct r600_transfer), 64);
703 
704 	sscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1));
705 	if (sscreen->force_aniso >= 0) {
706 		printf("radeonsi: Forcing anisotropy filter to %ix\n",
707 		       /* round down to a power of two */
708 		       1 << util_logbase2(sscreen->force_aniso));
709 	}
710 
711 	(void) mtx_init(&sscreen->aux_context_lock, mtx_plain);
712 	(void) mtx_init(&sscreen->gpu_load_mutex, mtx_plain);
713 
714 	if (!si_init_gs_info(sscreen) ||
715 	    !si_init_shader_cache(sscreen)) {
716 		FREE(sscreen);
717 		return NULL;
718 	}
719 
720 	si_disk_cache_create(sscreen);
721 
722 	/* Only enable as many threads as we have target machines, but at most
723 	 * the number of CPUs - 1 if there is more than one.
724 	 */
725 	num_threads = sysconf(_SC_NPROCESSORS_ONLN);
726 	num_threads = MAX2(1, num_threads - 1);
727 	num_compiler_threads = MIN2(num_threads, ARRAY_SIZE(sscreen->tm));
728 	num_compiler_threads_lowprio =
729 		MIN2(num_threads, ARRAY_SIZE(sscreen->tm_low_priority));
730 
731 	if (!util_queue_init(&sscreen->shader_compiler_queue, "si_shader",
732 			     32, num_compiler_threads,
733 			     UTIL_QUEUE_INIT_RESIZE_IF_FULL)) {
734 		si_destroy_shader_cache(sscreen);
735 		FREE(sscreen);
736 		return NULL;
737 	}
738 
739 	if (!util_queue_init(&sscreen->shader_compiler_queue_low_priority,
740 			     "si_shader_low",
741 			     32, num_compiler_threads_lowprio,
742 			     UTIL_QUEUE_INIT_RESIZE_IF_FULL |
743 			     UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY)) {
744 	       si_destroy_shader_cache(sscreen);
745 	       FREE(sscreen);
746 	       return NULL;
747 	}
748 
749 	si_handle_env_var_force_family(sscreen);
750 
751 	if (!debug_get_bool_option("RADEON_DISABLE_PERFCOUNTERS", false))
752 		si_init_perfcounters(sscreen);
753 
754 	/* Hawaii has a bug with offchip buffers > 256 that can be worked
755 	 * around by setting 4K granularity.
756 	 */
757 	sscreen->tess_offchip_block_dw_size =
758 		sscreen->info.family == CHIP_HAWAII ? 4096 : 8192;
759 
760 	/* The mere presense of CLEAR_STATE in the IB causes random GPU hangs
761 	 * on SI. */
762 	sscreen->has_clear_state = sscreen->info.chip_class >= CIK;
763 
764 	sscreen->has_distributed_tess =
765 		sscreen->info.chip_class >= VI &&
766 		sscreen->info.max_se >= 2;
767 
768 	sscreen->has_draw_indirect_multi =
769 		(sscreen->info.family >= CHIP_POLARIS10) ||
770 		(sscreen->info.chip_class == VI &&
771 		 sscreen->info.pfp_fw_version >= 121 &&
772 		 sscreen->info.me_fw_version >= 87) ||
773 		(sscreen->info.chip_class == CIK &&
774 		 sscreen->info.pfp_fw_version >= 211 &&
775 		 sscreen->info.me_fw_version >= 173) ||
776 		(sscreen->info.chip_class == SI &&
777 		 sscreen->info.pfp_fw_version >= 79 &&
778 		 sscreen->info.me_fw_version >= 142);
779 
780 	sscreen->has_out_of_order_rast = sscreen->info.chip_class >= VI &&
781 					 sscreen->info.max_se >= 2 &&
782 					 !(sscreen->debug_flags & DBG(NO_OUT_OF_ORDER));
783 	sscreen->assume_no_z_fights =
784 		driQueryOptionb(config->options, "radeonsi_assume_no_z_fights");
785 	sscreen->commutative_blend_add =
786 		driQueryOptionb(config->options, "radeonsi_commutative_blend_add");
787 	sscreen->clear_db_cache_before_clear =
788 		driQueryOptionb(config->options, "radeonsi_clear_db_cache_before_clear");
789 	sscreen->has_msaa_sample_loc_bug = (sscreen->info.family >= CHIP_POLARIS10 &&
790 					    sscreen->info.family <= CHIP_POLARIS12) ||
791 					   sscreen->info.family == CHIP_VEGA10 ||
792 					   sscreen->info.family == CHIP_RAVEN;
793 	sscreen->has_ls_vgpr_init_bug = sscreen->info.family == CHIP_VEGA10 ||
794 					sscreen->info.family == CHIP_RAVEN;
795 
796 	if (sscreen->debug_flags & DBG(DPBB)) {
797 		sscreen->dpbb_allowed = true;
798 	} else {
799 		/* Only enable primitive binning on Raven by default. */
800 		sscreen->dpbb_allowed = sscreen->info.family == CHIP_RAVEN &&
801 					!(sscreen->debug_flags & DBG(NO_DPBB));
802 	}
803 
804 	if (sscreen->debug_flags & DBG(DFSM)) {
805 		sscreen->dfsm_allowed = sscreen->dpbb_allowed;
806 	} else {
807 		sscreen->dfsm_allowed = sscreen->dpbb_allowed &&
808 					!(sscreen->debug_flags & DBG(NO_DFSM));
809 	}
810 
811 	/* While it would be nice not to have this flag, we are constrained
812 	 * by the reality that LLVM 5.0 doesn't have working VGPR indexing
813 	 * on GFX9.
814 	 */
815 	sscreen->llvm_has_working_vgpr_indexing = sscreen->info.chip_class <= VI;
816 
817 	/* Some chips have RB+ registers, but don't support RB+. Those must
818 	 * always disable it.
819 	 */
820 	if (sscreen->info.family == CHIP_STONEY ||
821 	    sscreen->info.chip_class >= GFX9) {
822 		sscreen->has_rbplus = true;
823 
824 		sscreen->rbplus_allowed =
825 			!(sscreen->debug_flags & DBG(NO_RB_PLUS)) &&
826 			(sscreen->info.family == CHIP_STONEY ||
827 			 sscreen->info.family == CHIP_RAVEN);
828 	}
829 
830 	sscreen->dcc_msaa_allowed =
831 		!(sscreen->debug_flags & DBG(NO_DCC_MSAA)) &&
832 		(sscreen->debug_flags & DBG(DCC_MSAA) ||
833 		 sscreen->info.chip_class == VI);
834 
835 	sscreen->cpdma_prefetch_writes_memory = sscreen->info.chip_class <= VI;
836 
837 	(void) mtx_init(&sscreen->shader_parts_mutex, mtx_plain);
838 	sscreen->use_monolithic_shaders =
839 		(sscreen->debug_flags & DBG(MONOLITHIC_SHADERS)) != 0;
840 
841 	sscreen->barrier_flags.cp_to_L2 = SI_CONTEXT_INV_SMEM_L1 |
842 					    SI_CONTEXT_INV_VMEM_L1;
843 	if (sscreen->info.chip_class <= VI) {
844 		sscreen->barrier_flags.cp_to_L2 |= SI_CONTEXT_INV_GLOBAL_L2;
845 		sscreen->barrier_flags.L2_to_cp |= SI_CONTEXT_WRITEBACK_GLOBAL_L2;
846 	}
847 
848 	if (debug_get_bool_option("RADEON_DUMP_SHADERS", false))
849 		sscreen->debug_flags |= DBG_ALL_SHADERS;
850 
851 	for (i = 0; i < num_compiler_threads; i++)
852 		sscreen->tm[i] = si_create_llvm_target_machine(sscreen);
853 	for (i = 0; i < num_compiler_threads_lowprio; i++)
854 		sscreen->tm_low_priority[i] = si_create_llvm_target_machine(sscreen);
855 
856 	/* Create the auxiliary context. This must be done last. */
857 	sscreen->aux_context = si_create_context(&sscreen->b, 0);
858 
859 	if (sscreen->debug_flags & DBG(TEST_DMA))
860 		si_test_dma(sscreen);
861 
862 	if (sscreen->debug_flags & (DBG(TEST_VMFAULT_CP) |
863 				      DBG(TEST_VMFAULT_SDMA) |
864 				      DBG(TEST_VMFAULT_SHADER)))
865 		si_test_vmfault(sscreen);
866 
867 	return &sscreen->b;
868 }
869