1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "si_pipe.h"
26 #include "si_shader_internal.h"
27 #include "sid.h"
28
get_rel_patch_id(struct si_shader_context * ctx)29 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
30 {
31 switch (ctx->stage) {
32 case MESA_SHADER_TESS_CTRL:
33 return si_unpack_param(ctx, ctx->args.tcs_rel_ids, 0, 8);
34
35 case MESA_SHADER_TESS_EVAL:
36 return ac_get_arg(&ctx->ac, ctx->tes_rel_patch_id);
37
38 default:
39 assert(0);
40 return NULL;
41 }
42 }
43
44 /* Tessellation shaders pass outputs to the next shader using LDS.
45 *
46 * LS outputs = TCS inputs
47 * TCS outputs = TES inputs
48 *
49 * The LDS layout is:
50 * - TCS inputs for patch 0
51 * - TCS inputs for patch 1
52 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
53 * - ...
54 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
55 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
56 * - TCS outputs for patch 1
57 * - Per-patch TCS outputs for patch 1
58 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
59 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
60 * - ...
61 *
62 * All three shaders VS(LS), TCS, TES share the same LDS space.
63 */
64
get_tcs_in_patch_stride(struct si_shader_context * ctx)65 static LLVMValueRef get_tcs_in_patch_stride(struct si_shader_context *ctx)
66 {
67 return si_unpack_param(ctx, ctx->vs_state_bits, 11, 13);
68 }
69
get_tcs_out_vertex_dw_stride_constant(struct si_shader_context * ctx)70 static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
71 {
72 assert(ctx->stage == MESA_SHADER_TESS_CTRL);
73
74 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
75 return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
76
77 return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
78 }
79
get_tcs_out_vertex_dw_stride(struct si_shader_context * ctx)80 static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
81 {
82 unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
83
84 return LLVMConstInt(ctx->ac.i32, stride, 0);
85 }
86
get_tcs_out_patch_stride(struct si_shader_context * ctx)87 static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
88 {
89 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
90 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 0, 13);
91
92 const struct si_shader_info *info = &ctx->shader->selector->info;
93 unsigned tcs_out_vertices = info->base.tess.tcs_vertices_out;
94 unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
95 unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
96 unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride + num_patch_outputs * 4;
97 return LLVMConstInt(ctx->ac.i32, patch_dw_stride, 0);
98 }
99
get_tcs_out_patch0_offset(struct si_shader_context * ctx)100 static LLVMValueRef get_tcs_out_patch0_offset(struct si_shader_context *ctx)
101 {
102 return LLVMBuildMul(ctx->ac.builder, si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 0, 16),
103 LLVMConstInt(ctx->ac.i32, 4, 0), "");
104 }
105
get_tcs_out_patch0_patch_data_offset(struct si_shader_context * ctx)106 static LLVMValueRef get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
107 {
108 return LLVMBuildMul(ctx->ac.builder, si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 16, 16),
109 LLVMConstInt(ctx->ac.i32, 4, 0), "");
110 }
111
get_tcs_in_current_patch_offset(struct si_shader_context * ctx)112 static LLVMValueRef get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
113 {
114 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
115 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
116
117 return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
118 }
119
get_tcs_out_current_patch_offset(struct si_shader_context * ctx)120 static LLVMValueRef get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
121 {
122 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
123 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
124 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
125
126 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_offset);
127 }
128
get_tcs_out_current_patch_data_offset(struct si_shader_context * ctx)129 static LLVMValueRef get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
130 {
131 LLVMValueRef patch0_patch_data_offset = get_tcs_out_patch0_patch_data_offset(ctx);
132 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
133 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
134
135 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_patch_data_offset);
136 }
137
get_num_tcs_out_vertices(struct si_shader_context * ctx)138 static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
139 {
140 unsigned tcs_out_vertices =
141 ctx->shader->selector ? ctx->shader->selector->info.base.tess.tcs_vertices_out
142 : 0;
143
144 /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
145 if (ctx->stage == MESA_SHADER_TESS_CTRL && tcs_out_vertices)
146 return LLVMConstInt(ctx->ac.i32, tcs_out_vertices, 0);
147
148 return si_unpack_param(ctx, ctx->tcs_offchip_layout, 6, 6);
149 }
150
get_tcs_in_vertex_dw_stride(struct si_shader_context * ctx)151 static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
152 {
153 unsigned stride;
154
155 switch (ctx->stage) {
156 case MESA_SHADER_VERTEX:
157 stride = ctx->shader->selector->lshs_vertex_stride / 4;
158 return LLVMConstInt(ctx->ac.i32, stride, 0);
159
160 case MESA_SHADER_TESS_CTRL:
161 if (ctx->screen->info.chip_class >= GFX9 && ctx->shader->is_monolithic) {
162 stride = ctx->shader->key.part.tcs.ls->lshs_vertex_stride / 4;
163 return LLVMConstInt(ctx->ac.i32, stride, 0);
164 }
165 return si_unpack_param(ctx, ctx->vs_state_bits, 24, 8);
166
167 default:
168 assert(0);
169 return NULL;
170 }
171 }
172
173 static LLVMValueRef
get_dw_address_from_generic_indices(struct si_shader_context * ctx,LLVMValueRef vertex_dw_stride,LLVMValueRef base_addr,LLVMValueRef vertex_index,LLVMValueRef param_index,ubyte name)174 get_dw_address_from_generic_indices(struct si_shader_context *ctx, LLVMValueRef vertex_dw_stride,
175 LLVMValueRef base_addr, LLVMValueRef vertex_index,
176 LLVMValueRef param_index, ubyte name)
177 {
178 if (vertex_dw_stride) {
179 base_addr = ac_build_imad(&ctx->ac, vertex_index, vertex_dw_stride, base_addr);
180 }
181
182 if (param_index) {
183 base_addr = ac_build_imad(&ctx->ac, param_index, LLVMConstInt(ctx->ac.i32, 4, 0), base_addr);
184 }
185
186 int param = name >= VARYING_SLOT_PATCH0 ||
187 name == VARYING_SLOT_TESS_LEVEL_INNER ||
188 name == VARYING_SLOT_TESS_LEVEL_OUTER
189 ? si_shader_io_get_unique_index_patch(name)
190 : si_shader_io_get_unique_index(name, false);
191
192 /* Add the base address of the element. */
193 return LLVMBuildAdd(ctx->ac.builder, base_addr, LLVMConstInt(ctx->ac.i32, param * 4, 0), "");
194 }
195
196 /* The offchip buffer layout for TCS->TES is
197 *
198 * - attribute 0 of patch 0 vertex 0
199 * - attribute 0 of patch 0 vertex 1
200 * - attribute 0 of patch 0 vertex 2
201 * ...
202 * - attribute 0 of patch 1 vertex 0
203 * - attribute 0 of patch 1 vertex 1
204 * ...
205 * - attribute 1 of patch 0 vertex 0
206 * - attribute 1 of patch 0 vertex 1
207 * ...
208 * - per patch attribute 0 of patch 0
209 * - per patch attribute 0 of patch 1
210 * ...
211 *
212 * Note that every attribute has 4 components.
213 */
get_tcs_tes_buffer_address(struct si_shader_context * ctx,LLVMValueRef rel_patch_id,LLVMValueRef vertex_index,LLVMValueRef param_index)214 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
215 LLVMValueRef rel_patch_id, LLVMValueRef vertex_index,
216 LLVMValueRef param_index)
217 {
218 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
219 LLVMValueRef param_stride, constant16;
220
221 vertices_per_patch = get_num_tcs_out_vertices(ctx);
222 num_patches = si_unpack_param(ctx, ctx->tcs_offchip_layout, 0, 6);
223 total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch, num_patches, "");
224
225 constant16 = LLVMConstInt(ctx->ac.i32, 16, 0);
226 if (vertex_index) {
227 base_addr = ac_build_imad(&ctx->ac, rel_patch_id, vertices_per_patch, vertex_index);
228 param_stride = total_vertices;
229 } else {
230 base_addr = rel_patch_id;
231 param_stride = num_patches;
232 }
233
234 base_addr = ac_build_imad(&ctx->ac, param_index, param_stride, base_addr);
235 base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
236
237 if (!vertex_index) {
238 LLVMValueRef patch_data_offset = si_unpack_param(ctx, ctx->tcs_offchip_layout, 12, 20);
239
240 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr, patch_data_offset, "");
241 }
242 return base_addr;
243 }
244
get_tcs_tes_buffer_address_from_generic_indices(struct si_shader_context * ctx,LLVMValueRef vertex_index,LLVMValueRef param_index,ubyte name)245 static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(struct si_shader_context *ctx,
246 LLVMValueRef vertex_index,
247 LLVMValueRef param_index,
248 ubyte name)
249 {
250 unsigned param_index_base;
251
252 param_index_base = name >= VARYING_SLOT_PATCH0 ||
253 name == VARYING_SLOT_TESS_LEVEL_INNER ||
254 name == VARYING_SLOT_TESS_LEVEL_OUTER
255 ? si_shader_io_get_unique_index_patch(name)
256 : si_shader_io_get_unique_index(name, false);
257
258 if (param_index) {
259 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
260 LLVMConstInt(ctx->ac.i32, param_index_base, 0), "");
261 } else {
262 param_index = LLVMConstInt(ctx->ac.i32, param_index_base, 0);
263 }
264
265 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), vertex_index, param_index);
266 }
267
buffer_load(struct si_shader_context * ctx,LLVMTypeRef type,unsigned swizzle,LLVMValueRef buffer,LLVMValueRef offset,LLVMValueRef base,bool can_speculate)268 static LLVMValueRef buffer_load(struct si_shader_context *ctx, LLVMTypeRef type, unsigned swizzle,
269 LLVMValueRef buffer, LLVMValueRef offset, LLVMValueRef base,
270 bool can_speculate)
271 {
272 LLVMValueRef value;
273 LLVMTypeRef vec_type = LLVMVectorType(type, 4);
274
275 if (swizzle == ~0) {
276 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset, 0, ac_glc,
277 can_speculate, false);
278
279 return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
280 }
281
282 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset, 0, ac_glc,
283 can_speculate, false);
284
285 value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
286 return LLVMBuildExtractElement(ctx->ac.builder, value, LLVMConstInt(ctx->ac.i32, swizzle, 0),
287 "");
288 }
289
290 /**
291 * Load from LSHS LDS storage.
292 *
293 * \param type output value type
294 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
295 * \param dw_addr address in dwords
296 */
lshs_lds_load(struct si_shader_context * ctx,LLVMTypeRef type,unsigned swizzle,LLVMValueRef dw_addr)297 static LLVMValueRef lshs_lds_load(struct si_shader_context *ctx, LLVMTypeRef type, unsigned swizzle,
298 LLVMValueRef dw_addr)
299 {
300 LLVMValueRef value;
301
302 if (swizzle == ~0) {
303 LLVMValueRef values[4];
304
305 for (unsigned chan = 0; chan < 4; chan++)
306 values[chan] = lshs_lds_load(ctx, type, chan, dw_addr);
307
308 return ac_build_gather_values(&ctx->ac, values, 4);
309 }
310
311 dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr, LLVMConstInt(ctx->ac.i32, swizzle, 0), "");
312 value = ac_lds_load(&ctx->ac, dw_addr);
313 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
314 }
315
316 /**
317 * Store to LSHS LDS storage.
318 *
319 * \param swizzle offset (typically 0..3)
320 * \param dw_addr address in dwords
321 * \param value value to store
322 */
lshs_lds_store(struct si_shader_context * ctx,unsigned dw_offset_imm,LLVMValueRef dw_addr,LLVMValueRef value)323 static void lshs_lds_store(struct si_shader_context *ctx, unsigned dw_offset_imm,
324 LLVMValueRef dw_addr, LLVMValueRef value)
325 {
326 dw_addr =
327 LLVMBuildAdd(ctx->ac.builder, dw_addr, LLVMConstInt(ctx->ac.i32, dw_offset_imm, 0), "");
328
329 ac_lds_store(&ctx->ac, dw_addr, value);
330 }
331
332 enum si_tess_ring
333 {
334 TCS_FACTOR_RING,
335 TESS_OFFCHIP_RING_TCS,
336 TESS_OFFCHIP_RING_TES,
337 };
338
get_tess_ring_descriptor(struct si_shader_context * ctx,enum si_tess_ring ring)339 static LLVMValueRef get_tess_ring_descriptor(struct si_shader_context *ctx, enum si_tess_ring ring)
340 {
341 LLVMBuilderRef builder = ctx->ac.builder;
342 LLVMValueRef addr = ac_get_arg(
343 &ctx->ac, ring == TESS_OFFCHIP_RING_TES ? ctx->tes_offchip_addr : ctx->tcs_out_lds_layout);
344
345 /* TCS only receives high 13 bits of the address. */
346 if (ring == TESS_OFFCHIP_RING_TCS || ring == TCS_FACTOR_RING) {
347 addr = LLVMBuildAnd(builder, addr, LLVMConstInt(ctx->ac.i32, 0xfff80000, 0), "");
348 }
349
350 if (ring == TCS_FACTOR_RING) {
351 unsigned tf_offset = ctx->screen->tess_offchip_ring_size;
352 addr = LLVMBuildAdd(builder, addr, LLVMConstInt(ctx->ac.i32, tf_offset, 0), "");
353 }
354
355 uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
356 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
357
358 if (ctx->screen->info.chip_class >= GFX10)
359 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
360 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1);
361 else
362 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
363 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
364
365 LLVMValueRef desc[4];
366 desc[0] = addr;
367 desc[1] = LLVMConstInt(ctx->ac.i32, S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
368 desc[2] = LLVMConstInt(ctx->ac.i32, 0xffffffff, 0);
369 desc[3] = LLVMConstInt(ctx->ac.i32, rsrc3, false);
370
371 return ac_build_gather_values(&ctx->ac, desc, 4);
372 }
373
si_llvm_preload_tes_rings(struct si_shader_context * ctx)374 void si_llvm_preload_tes_rings(struct si_shader_context *ctx)
375 {
376 ctx->tess_offchip_ring = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TES);
377 }
378
si_nir_load_tcs_varyings(struct ac_shader_abi * abi,LLVMTypeRef type,LLVMValueRef vertex_index,LLVMValueRef param_index,unsigned driver_location,unsigned component,unsigned num_components,bool load_input)379 static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi, LLVMTypeRef type,
380 LLVMValueRef vertex_index, LLVMValueRef param_index,
381 unsigned driver_location, unsigned component,
382 unsigned num_components, bool load_input)
383 {
384 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
385 struct si_shader_info *info = &ctx->shader->selector->info;
386 LLVMValueRef dw_addr, stride;
387 ubyte semantic;
388
389 if (load_input) {
390 semantic = info->input_semantic[driver_location];
391 } else {
392 semantic = info->output_semantic[driver_location];
393 }
394
395 bool is_patch = vertex_index == NULL;
396 assert((semantic >= VARYING_SLOT_PATCH0 ||
397 semantic == VARYING_SLOT_TESS_LEVEL_INNER ||
398 semantic == VARYING_SLOT_TESS_LEVEL_OUTER) == is_patch);
399
400 if (load_input) {
401 stride = get_tcs_in_vertex_dw_stride(ctx);
402 dw_addr = get_tcs_in_current_patch_offset(ctx);
403 } else {
404 if (is_patch) {
405 stride = NULL;
406 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
407 } else {
408 stride = get_tcs_out_vertex_dw_stride(ctx);
409 dw_addr = get_tcs_out_current_patch_offset(ctx);
410 }
411 }
412
413 if (!param_index) {
414 param_index = ctx->ac.i32_0;
415 }
416
417 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr, vertex_index, param_index,
418 semantic);
419
420 LLVMValueRef value[4];
421 for (unsigned i = component; i < component + num_components; i++)
422 value[i] = lshs_lds_load(ctx, type, i, dw_addr);
423
424 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
425 }
426
si_nir_load_input_tes(struct ac_shader_abi * abi,LLVMTypeRef type,LLVMValueRef vertex_index,LLVMValueRef param_index,unsigned driver_location,unsigned component,unsigned num_components,bool load_input)427 static LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi, LLVMTypeRef type,
428 LLVMValueRef vertex_index, LLVMValueRef param_index,
429 unsigned driver_location, unsigned component,
430 unsigned num_components,
431 bool load_input)
432 {
433 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
434 struct si_shader_info *info = &ctx->shader->selector->info;
435 LLVMValueRef base, addr;
436
437 ubyte semantic = info->input_semantic[driver_location];
438
439 assert((semantic >= VARYING_SLOT_PATCH0 ||
440 semantic == VARYING_SLOT_TESS_LEVEL_INNER ||
441 semantic == VARYING_SLOT_TESS_LEVEL_OUTER) == (vertex_index == NULL));
442
443 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
444
445 if (!param_index) {
446 param_index = ctx->ac.i32_0;
447 }
448
449 addr =
450 get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index, param_index, semantic);
451
452 /* TODO: This will generate rather ordinary llvm code, although it
453 * should be easy for the optimiser to fix up. In future we might want
454 * to refactor buffer_load().
455 */
456 LLVMValueRef value[4];
457 for (unsigned i = component; i < component + num_components; i++)
458 value[i] = buffer_load(ctx, type, i, ctx->tess_offchip_ring, base, addr, true);
459
460 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
461 }
462
si_nir_store_output_tcs(struct ac_shader_abi * abi,LLVMValueRef vertex_index,LLVMValueRef param_index,LLVMValueRef src,unsigned writemask,unsigned component,unsigned location,unsigned driver_location)463 static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
464 LLVMValueRef vertex_index, LLVMValueRef param_index,
465 LLVMValueRef src, unsigned writemask,
466 unsigned component, unsigned location, unsigned driver_location)
467 {
468 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
469 struct si_shader_info *info = &ctx->shader->selector->info;
470 LLVMValueRef dw_addr, stride;
471 LLVMValueRef buffer, base, addr;
472 LLVMValueRef values[8];
473 bool is_tess_factor = false, is_tess_inner = false;
474
475 ubyte semantic = info->output_semantic[driver_location];
476
477 bool is_const = !param_index;
478 if (!param_index)
479 param_index = ctx->ac.i32_0;
480
481 const bool is_patch = vertex_index == NULL;
482
483 /* Invalid SPIR-V can cause this. */
484 if ((semantic >= VARYING_SLOT_PATCH0 || semantic == VARYING_SLOT_TESS_LEVEL_INNER ||
485 semantic == VARYING_SLOT_TESS_LEVEL_OUTER) != is_patch)
486 return;
487
488 if (!is_patch) {
489 stride = get_tcs_out_vertex_dw_stride(ctx);
490 dw_addr = get_tcs_out_current_patch_offset(ctx);
491 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr, vertex_index, param_index,
492 semantic);
493 } else {
494 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
495 dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr, vertex_index, param_index,
496 semantic);
497
498 if (is_const) {
499 int semantic = info->output_semantic[driver_location];
500
501 /* Always write tess factors into LDS for the TCS epilog. */
502 if (semantic == VARYING_SLOT_TESS_LEVEL_INNER ||
503 semantic == VARYING_SLOT_TESS_LEVEL_OUTER) {
504 is_tess_factor = true;
505 is_tess_inner = semantic == VARYING_SLOT_TESS_LEVEL_INNER;
506 }
507 }
508 }
509
510 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
511
512 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
513
514 addr =
515 get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index, param_index, semantic);
516
517 for (unsigned chan = component; chan < 4; chan++) {
518 if (!(writemask & (1 << chan)))
519 continue;
520 LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
521
522 /* Skip LDS stores if there is no LDS read of this output. */
523 if (info->output_readmask[driver_location] & (1 << chan) ||
524 /* The epilog reads LDS if invocation 0 doesn't define tess factors. */
525 (is_tess_factor &&
526 !ctx->shader->selector->info.tessfactors_are_def_in_all_invocs))
527 lshs_lds_store(ctx, chan, dw_addr, value);
528
529 value = ac_to_integer(&ctx->ac, value);
530 values[chan] = value;
531
532 if (writemask != 0xF && !is_tess_factor) {
533 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1, addr, base,
534 4 * chan, ac_glc);
535 }
536
537 /* Write tess factors into VGPRs for the epilog. */
538 if (is_tess_factor && ctx->shader->selector->info.tessfactors_are_def_in_all_invocs) {
539 if (!is_tess_inner) {
540 LLVMBuildStore(ctx->ac.builder, value, /* outer */
541 ctx->invoc0_tess_factors[chan]);
542 } else if (chan < 2) {
543 LLVMBuildStore(ctx->ac.builder, value, /* inner */
544 ctx->invoc0_tess_factors[4 + chan]);
545 }
546 }
547 }
548
549 if (writemask == 0xF && !is_tess_factor) {
550 LLVMValueRef value = ac_build_gather_values(&ctx->ac, values, 4);
551 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr, base, 0, ac_glc);
552 }
553 }
554
si_load_tess_coord(struct ac_shader_abi * abi)555 static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
556 {
557 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
558 LLVMValueRef coord[4] = {ac_get_arg(&ctx->ac, ctx->tes_u), ac_get_arg(&ctx->ac, ctx->tes_v),
559 ctx->ac.f32_0, ctx->ac.f32_0};
560
561 /* For triangles, the vector should be (u, v, 1-u-v). */
562 if (ctx->shader->selector->info.base.tess.primitive_mode == GL_TRIANGLES) {
563 coord[2] = LLVMBuildFSub(ctx->ac.builder, ctx->ac.f32_1,
564 LLVMBuildFAdd(ctx->ac.builder, coord[0], coord[1], ""), "");
565 }
566 return ac_build_gather_values(&ctx->ac, coord, 4);
567 }
568
load_tess_level(struct si_shader_context * ctx,unsigned semantic)569 static LLVMValueRef load_tess_level(struct si_shader_context *ctx, unsigned semantic)
570 {
571 LLVMValueRef base, addr;
572
573 int param = si_shader_io_get_unique_index_patch(semantic);
574
575 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
576 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
577 LLVMConstInt(ctx->ac.i32, param, 0));
578
579 return buffer_load(ctx, ctx->ac.f32, ~0, ctx->tess_offchip_ring, base, addr, true);
580 }
581
load_tess_level_default(struct si_shader_context * ctx,unsigned sysval)582 static LLVMValueRef load_tess_level_default(struct si_shader_context *ctx, unsigned sysval)
583 {
584 LLVMValueRef buf, slot, val[4];
585 int i, offset;
586
587 slot = LLVMConstInt(ctx->ac.i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
588 buf = ac_get_arg(&ctx->ac, ctx->rw_buffers);
589 buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
590 offset = sysval == SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT ? 4 : 0;
591
592 for (i = 0; i < 4; i++)
593 val[i] = si_buffer_load_const(ctx, buf, LLVMConstInt(ctx->ac.i32, (offset + i) * 4, 0));
594 return ac_build_gather_values(&ctx->ac, val, 4);
595 }
596
si_load_tess_level(struct ac_shader_abi * abi,unsigned varying_id,bool load_default_state)597 static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi, unsigned varying_id,
598 bool load_default_state)
599 {
600 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
601 unsigned semantic;
602
603 if (load_default_state) {
604 switch (varying_id) {
605 case VARYING_SLOT_TESS_LEVEL_INNER:
606 semantic = SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT;
607 break;
608 case VARYING_SLOT_TESS_LEVEL_OUTER:
609 semantic = SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT;
610 break;
611 default:
612 unreachable("unknown tess level");
613 }
614 return load_tess_level_default(ctx, semantic);
615 }
616
617 switch (varying_id) {
618 case VARYING_SLOT_TESS_LEVEL_INNER:
619 semantic = VARYING_SLOT_TESS_LEVEL_INNER;
620 break;
621 case VARYING_SLOT_TESS_LEVEL_OUTER:
622 semantic = VARYING_SLOT_TESS_LEVEL_OUTER;
623 break;
624 default:
625 unreachable("unknown tess level");
626 }
627
628 return load_tess_level(ctx, semantic);
629 }
630
si_load_patch_vertices_in(struct ac_shader_abi * abi)631 static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
632 {
633 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
634 if (ctx->stage == MESA_SHADER_TESS_CTRL)
635 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 13, 6);
636 else if (ctx->stage == MESA_SHADER_TESS_EVAL)
637 return get_num_tcs_out_vertices(ctx);
638 else
639 unreachable("invalid shader stage for VERTICESIN");
640 }
641
642 /**
643 * Forward all outputs from the vertex shader to the TES. This is only used
644 * for the fixed function TCS.
645 */
si_copy_tcs_inputs(struct si_shader_context * ctx)646 static void si_copy_tcs_inputs(struct si_shader_context *ctx)
647 {
648 LLVMValueRef invocation_id, buffer, buffer_offset;
649 LLVMValueRef lds_vertex_stride, lds_base;
650 uint64_t inputs;
651
652 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
653 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
654 buffer_offset = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
655
656 lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
657 lds_base = get_tcs_in_current_patch_offset(ctx);
658 lds_base = ac_build_imad(&ctx->ac, invocation_id, lds_vertex_stride, lds_base);
659
660 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
661 while (inputs) {
662 unsigned i = u_bit_scan64(&inputs);
663
664 LLVMValueRef lds_ptr =
665 LLVMBuildAdd(ctx->ac.builder, lds_base, LLVMConstInt(ctx->ac.i32, 4 * i, 0), "");
666
667 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(
668 ctx, get_rel_patch_id(ctx), invocation_id, LLVMConstInt(ctx->ac.i32, i, 0));
669
670 LLVMValueRef value = lshs_lds_load(ctx, ctx->ac.i32, ~0, lds_ptr);
671
672 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr, buffer_offset, 0,
673 ac_glc);
674 }
675 }
676
si_write_tess_factors(struct si_shader_context * ctx,LLVMValueRef rel_patch_id,LLVMValueRef invocation_id,LLVMValueRef tcs_out_current_patch_data_offset,LLVMValueRef invoc0_tf_outer[4],LLVMValueRef invoc0_tf_inner[2])677 static void si_write_tess_factors(struct si_shader_context *ctx, LLVMValueRef rel_patch_id,
678 LLVMValueRef invocation_id,
679 LLVMValueRef tcs_out_current_patch_data_offset,
680 LLVMValueRef invoc0_tf_outer[4], LLVMValueRef invoc0_tf_inner[2])
681 {
682 struct si_shader *shader = ctx->shader;
683 unsigned tess_inner_index, tess_outer_index;
684 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
685 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
686 unsigned stride, outer_comps, inner_comps, i, offset;
687
688 /* Add a barrier before loading tess factors from LDS. */
689 if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
690 si_llvm_emit_barrier(ctx);
691
692 /* Do this only for invocation 0, because the tess levels are per-patch,
693 * not per-vertex.
694 *
695 * This can't jump, because invocation 0 executes this. It should
696 * at least mask out the loads and stores for other invocations.
697 */
698 ac_build_ifcc(&ctx->ac,
699 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ, invocation_id, ctx->ac.i32_0, ""), 6503);
700
701 /* Determine the layout of one tess factor element in the buffer. */
702 switch (shader->key.part.tcs.epilog.prim_mode) {
703 case GL_LINES:
704 stride = 2; /* 2 dwords, 1 vec2 store */
705 outer_comps = 2;
706 inner_comps = 0;
707 break;
708 case GL_TRIANGLES:
709 stride = 4; /* 4 dwords, 1 vec4 store */
710 outer_comps = 3;
711 inner_comps = 1;
712 break;
713 case GL_QUADS:
714 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
715 outer_comps = 4;
716 inner_comps = 2;
717 break;
718 default:
719 assert(0);
720 return;
721 }
722
723 for (i = 0; i < 4; i++) {
724 inner[i] = LLVMGetUndef(ctx->ac.i32);
725 outer[i] = LLVMGetUndef(ctx->ac.i32);
726 }
727
728 if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
729 /* Tess factors are in VGPRs. */
730 for (i = 0; i < outer_comps; i++)
731 outer[i] = out[i] = invoc0_tf_outer[i];
732 for (i = 0; i < inner_comps; i++)
733 inner[i] = out[outer_comps + i] = invoc0_tf_inner[i];
734 } else {
735 /* Load tess_inner and tess_outer from LDS.
736 * Any invocation can write them, so we can't get them from a temporary.
737 */
738 tess_inner_index = si_shader_io_get_unique_index_patch(VARYING_SLOT_TESS_LEVEL_INNER);
739 tess_outer_index = si_shader_io_get_unique_index_patch(VARYING_SLOT_TESS_LEVEL_OUTER);
740
741 lds_base = tcs_out_current_patch_data_offset;
742 lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
743 LLVMConstInt(ctx->ac.i32, tess_inner_index * 4, 0), "");
744 lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
745 LLVMConstInt(ctx->ac.i32, tess_outer_index * 4, 0), "");
746
747 for (i = 0; i < outer_comps; i++) {
748 outer[i] = out[i] = lshs_lds_load(ctx, ctx->ac.i32, i, lds_outer);
749 }
750 for (i = 0; i < inner_comps; i++) {
751 inner[i] = out[outer_comps + i] = lshs_lds_load(ctx, ctx->ac.i32, i, lds_inner);
752 }
753 }
754
755 if (shader->key.part.tcs.epilog.prim_mode == GL_LINES) {
756 /* For isolines, the hardware expects tess factors in the
757 * reverse order from what NIR specifies.
758 */
759 LLVMValueRef tmp = out[0];
760 out[0] = out[1];
761 out[1] = tmp;
762 }
763
764 /* Convert the outputs to vectors for stores. */
765 vec0 = ac_build_gather_values(&ctx->ac, out, MIN2(stride, 4));
766 vec1 = NULL;
767
768 if (stride > 4)
769 vec1 = ac_build_gather_values(&ctx->ac, out + 4, stride - 4);
770
771 /* Get the buffer. */
772 buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
773
774 /* Get the offset. */
775 tf_base = ac_get_arg(&ctx->ac, ctx->tcs_factor_offset);
776 byteoffset =
777 LLVMBuildMul(ctx->ac.builder, rel_patch_id, LLVMConstInt(ctx->ac.i32, 4 * stride, 0), "");
778
779 ac_build_ifcc(&ctx->ac,
780 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ, rel_patch_id, ctx->ac.i32_0, ""), 6504);
781
782 /* Store the dynamic HS control word. */
783 offset = 0;
784 if (ctx->screen->info.chip_class <= GFX8) {
785 ac_build_buffer_store_dword(&ctx->ac, buffer, LLVMConstInt(ctx->ac.i32, 0x80000000, 0), 1,
786 ctx->ac.i32_0, tf_base, offset, ac_glc);
787 offset += 4;
788 }
789
790 ac_build_endif(&ctx->ac, 6504);
791
792 /* Store the tessellation factors. */
793 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0, MIN2(stride, 4), byteoffset, tf_base, offset,
794 ac_glc);
795 offset += 16;
796 if (vec1)
797 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1, stride - 4, byteoffset, tf_base, offset,
798 ac_glc);
799
800 /* Store the tess factors into the offchip buffer if TES reads them. */
801 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
802 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
803 LLVMValueRef tf_inner_offset;
804 unsigned param_outer, param_inner;
805
806 buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
807 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
808
809 param_outer = si_shader_io_get_unique_index_patch(VARYING_SLOT_TESS_LEVEL_OUTER);
810 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
811 LLVMConstInt(ctx->ac.i32, param_outer, 0));
812
813 unsigned outer_vec_size = ac_has_vec3_support(ctx->screen->info.chip_class, false)
814 ? outer_comps
815 : util_next_power_of_two(outer_comps);
816 outer_vec = ac_build_gather_values(&ctx->ac, outer, outer_vec_size);
817
818 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec, outer_comps, tf_outer_offset, base, 0,
819 ac_glc);
820 if (inner_comps) {
821 param_inner = si_shader_io_get_unique_index_patch(VARYING_SLOT_TESS_LEVEL_INNER);
822 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
823 LLVMConstInt(ctx->ac.i32, param_inner, 0));
824
825 inner_vec =
826 inner_comps == 1 ? inner[0] : ac_build_gather_values(&ctx->ac, inner, inner_comps);
827 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec, inner_comps, tf_inner_offset, base,
828 0, ac_glc);
829 }
830 }
831
832 ac_build_endif(&ctx->ac, 6503);
833 }
834
835 /* This only writes the tessellation factor levels. */
si_llvm_emit_tcs_epilogue(struct ac_shader_abi * abi,unsigned max_outputs,LLVMValueRef * addrs)836 static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi, unsigned max_outputs,
837 LLVMValueRef *addrs)
838 {
839 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
840 LLVMBuilderRef builder = ctx->ac.builder;
841 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
842
843 si_copy_tcs_inputs(ctx);
844
845 rel_patch_id = get_rel_patch_id(ctx);
846 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
847 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
848
849 if (ctx->screen->info.chip_class >= GFX9) {
850 LLVMBasicBlockRef blocks[2] = {LLVMGetInsertBlock(builder), ctx->merged_wrap_if_entry_block};
851 LLVMValueRef values[2];
852
853 ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
854
855 values[0] = rel_patch_id;
856 values[1] = LLVMGetUndef(ctx->ac.i32);
857 rel_patch_id = ac_build_phi(&ctx->ac, ctx->ac.i32, 2, values, blocks);
858
859 values[0] = tf_lds_offset;
860 values[1] = LLVMGetUndef(ctx->ac.i32);
861 tf_lds_offset = ac_build_phi(&ctx->ac, ctx->ac.i32, 2, values, blocks);
862
863 values[0] = invocation_id;
864 values[1] = ctx->ac.i32_1; /* cause the epilog to skip threads */
865 invocation_id = ac_build_phi(&ctx->ac, ctx->ac.i32, 2, values, blocks);
866 }
867
868 /* Return epilog parameters from this function. */
869 LLVMValueRef ret = ctx->return_value;
870 unsigned vgpr;
871
872 if (ctx->screen->info.chip_class >= GFX9) {
873 ret =
874 si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout, 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
875 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout, 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
876 /* Tess offchip and tess factor offsets are at the beginning. */
877 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
878 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
879 vgpr = 8 + GFX9_SGPR_TCS_OUT_LAYOUT + 1;
880 } else {
881 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout, GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
882 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout, GFX6_SGPR_TCS_OUT_LAYOUT);
883 /* Tess offchip and tess factor offsets are after user SGPRs. */
884 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, GFX6_TCS_NUM_USER_SGPR);
885 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, GFX6_TCS_NUM_USER_SGPR + 1);
886 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
887 }
888
889 /* VGPRs */
890 rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
891 invocation_id = ac_to_float(&ctx->ac, invocation_id);
892 tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
893
894 /* Leave a hole corresponding to the two input VGPRs. This ensures that
895 * the invocation_id output does not alias the tcs_rel_ids input,
896 * which saves a V_MOV on gfx9.
897 */
898 vgpr += 2;
899
900 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
901 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
902
903 if (ctx->shader->selector->info.tessfactors_are_def_in_all_invocs) {
904 vgpr++; /* skip the tess factor LDS offset */
905 for (unsigned i = 0; i < 6; i++) {
906 LLVMValueRef value = LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
907 value = ac_to_float(&ctx->ac, value);
908 ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
909 }
910 } else {
911 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
912 }
913 ctx->return_value = ret;
914 }
915
916 /* Pass TCS inputs from LS to TCS on GFX9. */
si_set_ls_return_value_for_tcs(struct si_shader_context * ctx)917 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
918 {
919 LLVMValueRef ret = ctx->return_value;
920
921 ret = si_insert_input_ptr(ctx, ret, ctx->other_const_and_shader_buffers, 0);
922 ret = si_insert_input_ptr(ctx, ret, ctx->other_samplers_and_images, 1);
923 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
924 ret = si_insert_input_ret(ctx, ret, ctx->merged_wave_info, 3);
925 ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
926 ret = si_insert_input_ret(ctx, ret, ctx->merged_scratch_offset, 5);
927
928 ret = si_insert_input_ptr(ctx, ret, ctx->rw_buffers, 8 + SI_SGPR_RW_BUFFERS);
929 ret = si_insert_input_ptr(ctx, ret, ctx->bindless_samplers_and_images,
930 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
931
932 ret = si_insert_input_ret(ctx, ret, ctx->vs_state_bits, 8 + SI_SGPR_VS_STATE_BITS);
933
934 ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout, 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
935 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_offsets, 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
936 ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout, 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
937
938 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
939 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
940 ac_to_float(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args.tcs_patch_id)),
941 vgpr++, "");
942 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
943 ac_to_float(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args.tcs_rel_ids)),
944 vgpr++, "");
945 ctx->return_value = ret;
946 }
947
si_llvm_emit_ls_epilogue(struct ac_shader_abi * abi,unsigned max_outputs,LLVMValueRef * addrs)948 void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi, unsigned max_outputs, LLVMValueRef *addrs)
949 {
950 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
951 struct si_shader *shader = ctx->shader;
952 struct si_shader_info *info = &shader->selector->info;
953 unsigned i, chan;
954 LLVMValueRef vertex_id = ac_get_arg(&ctx->ac, ctx->rel_auto_id);
955 LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
956 LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id, vertex_dw_stride, "");
957
958 /* Write outputs to LDS. The next shader (TCS aka HS) will read
959 * its inputs from it. */
960 for (i = 0; i < info->num_outputs; i++) {
961 unsigned semantic = info->output_semantic[i];
962
963 /* The ARB_shader_viewport_layer_array spec contains the
964 * following issue:
965 *
966 * 2) What happens if gl_ViewportIndex or gl_Layer is
967 * written in the vertex shader and a geometry shader is
968 * present?
969 *
970 * RESOLVED: The value written by the last vertex processing
971 * stage is used. If the last vertex processing stage
972 * (vertex, tessellation evaluation or geometry) does not
973 * statically assign to gl_ViewportIndex or gl_Layer, index
974 * or layer zero is assumed.
975 *
976 * So writes to those outputs in VS-as-LS are simply ignored.
977 */
978 if (semantic == VARYING_SLOT_LAYER || semantic == VARYING_SLOT_VIEWPORT)
979 continue;
980
981 int param = si_shader_io_get_unique_index(semantic, false);
982 LLVMValueRef dw_addr =
983 LLVMBuildAdd(ctx->ac.builder, base_dw_addr, LLVMConstInt(ctx->ac.i32, param * 4, 0), "");
984
985 for (chan = 0; chan < 4; chan++) {
986 if (!(info->output_usagemask[i] & (1 << chan)))
987 continue;
988
989 lshs_lds_store(ctx, chan, dw_addr,
990 LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
991 }
992 }
993
994 if (ctx->screen->info.chip_class >= GFX9)
995 si_set_ls_return_value_for_tcs(ctx);
996 }
997
998 /**
999 * Compile the TCS epilog function. This writes tesselation factors to memory
1000 * based on the output primitive type of the tesselator (determined by TES).
1001 */
si_llvm_build_tcs_epilog(struct si_shader_context * ctx,union si_shader_part_key * key)1002 void si_llvm_build_tcs_epilog(struct si_shader_context *ctx, union si_shader_part_key *key)
1003 {
1004 memset(&ctx->args, 0, sizeof(ctx->args));
1005
1006 if (ctx->screen->info.chip_class >= GFX9) {
1007 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1008 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1009 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
1010 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* wave info */
1011 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_factor_offset);
1012 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1013 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1014 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1015 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1016 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1017 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1018 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1019 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1020 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1021 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1022 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1023 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
1024 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1025 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_layout);
1026 } else {
1027 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1028 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1029 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1030 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1031 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_layout);
1032 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1033 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_out_lds_layout);
1034 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
1035 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_offchip_offset);
1036 ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, &ctx->tcs_factor_offset);
1037 }
1038
1039 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
1040 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
1041 struct ac_arg rel_patch_id; /* patch index within the wave (REL_PATCH_ID) */
1042 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &rel_patch_id);
1043 struct ac_arg invocation_id; /* invocation ID within the patch */
1044 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &invocation_id);
1045 struct ac_arg
1046 tcs_out_current_patch_data_offset; /* LDS offset where tess factors should be loaded from */
1047 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &tcs_out_current_patch_data_offset);
1048
1049 struct ac_arg tess_factors[6];
1050 for (unsigned i = 0; i < 6; i++)
1051 ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &tess_factors[i]);
1052
1053 /* Create the function. */
1054 si_llvm_create_func(ctx, "tcs_epilog", NULL, 0, ctx->screen->info.chip_class >= GFX7 ? 128 : 0);
1055 ac_declare_lds_as_pointer(&ctx->ac);
1056
1057 LLVMValueRef invoc0_tess_factors[6];
1058 for (unsigned i = 0; i < 6; i++)
1059 invoc0_tess_factors[i] = ac_get_arg(&ctx->ac, tess_factors[i]);
1060
1061 si_write_tess_factors(ctx, ac_get_arg(&ctx->ac, rel_patch_id),
1062 ac_get_arg(&ctx->ac, invocation_id),
1063 ac_get_arg(&ctx->ac, tcs_out_current_patch_data_offset),
1064 invoc0_tess_factors, invoc0_tess_factors + 4);
1065
1066 LLVMBuildRetVoid(ctx->ac.builder);
1067 }
1068
si_llvm_init_tcs_callbacks(struct si_shader_context * ctx)1069 void si_llvm_init_tcs_callbacks(struct si_shader_context *ctx)
1070 {
1071 ctx->abi.load_tess_varyings = si_nir_load_tcs_varyings;
1072 ctx->abi.load_tess_level = si_load_tess_level;
1073 ctx->abi.store_tcs_outputs = si_nir_store_output_tcs;
1074 ctx->abi.emit_outputs = si_llvm_emit_tcs_epilogue;
1075 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
1076 }
1077
si_llvm_init_tes_callbacks(struct si_shader_context * ctx,bool ngg_cull_shader)1078 void si_llvm_init_tes_callbacks(struct si_shader_context *ctx, bool ngg_cull_shader)
1079 {
1080 ctx->abi.load_tess_varyings = si_nir_load_input_tes;
1081 ctx->abi.load_tess_coord = si_load_tess_coord;
1082 ctx->abi.load_tess_level = si_load_tess_level;
1083 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
1084
1085 if (ctx->shader->key.as_es)
1086 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
1087 else if (ngg_cull_shader)
1088 ctx->abi.emit_outputs = gfx10_emit_ngg_culling_epilogue;
1089 else if (ctx->shader->key.as_ngg)
1090 ctx->abi.emit_outputs = gfx10_emit_ngg_epilogue;
1091 else
1092 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
1093 }
1094