1 /* -*- mesa-c++ -*-
2 *
3 * Copyright (c) 2022 Collabora LTD
4 *
5 * Author: Gert Wollny <gert.wollny@collabora.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * on the rights to use, copy, modify, merge, publish, distribute, sub
11 * license, and/or sell copies of the Software, and to permit persons to whom
12 * the Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27 #include "sfn_nir.h"
28
r600_lower_tess_io_filter(const nir_instr * instr,gl_shader_stage stage)29 bool r600_lower_tess_io_filter(const nir_instr *instr, gl_shader_stage stage)
30 {
31 if (instr->type != nir_instr_type_intrinsic)
32 return false;
33
34 nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr);
35 switch (op->intrinsic) {
36 case nir_intrinsic_load_input:
37 return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL;
38 case nir_intrinsic_load_output:
39 case nir_intrinsic_load_per_vertex_input:
40 case nir_intrinsic_load_per_vertex_output:
41 case nir_intrinsic_store_per_vertex_output:
42 case nir_intrinsic_load_patch_vertices_in:
43 case nir_intrinsic_load_tess_level_outer:
44 case nir_intrinsic_load_tess_level_inner:
45 return true;
46 case nir_intrinsic_store_output:
47 return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_VERTEX;
48 default:
49 ;
50 }
51 return false;
52 }
53
54 static nir_ssa_def *
emit_load_param_base(nir_builder * b,nir_intrinsic_op op)55 emit_load_param_base(nir_builder *b, nir_intrinsic_op op)
56 {
57 nir_intrinsic_instr *result = nir_intrinsic_instr_create(b->shader, op);
58 nir_ssa_dest_init(&result->instr, &result->dest,
59 4, 32, NULL);
60 nir_builder_instr_insert(b, &result->instr);
61 return &result->dest.ssa;
62 }
63
get_tcs_varying_offset(nir_intrinsic_instr * op)64 static int get_tcs_varying_offset(nir_intrinsic_instr *op)
65 {
66 unsigned location = nir_intrinsic_io_semantics(op).location;
67
68 switch (location) {
69 case VARYING_SLOT_POS:
70 return 0;
71 case VARYING_SLOT_PSIZ:
72 return 0x10;
73 case VARYING_SLOT_CLIP_DIST0:
74 return 0x20;
75 case VARYING_SLOT_CLIP_DIST1:
76 return 0x30;
77 case VARYING_SLOT_COL0:
78 return 0x40;
79 case VARYING_SLOT_COL1:
80 return 0x50;
81 case VARYING_SLOT_BFC0:
82 return 0x60;
83 case VARYING_SLOT_BFC1:
84 return 0x70;
85 case VARYING_SLOT_CLIP_VERTEX:
86 return 0x80;
87 case VARYING_SLOT_TESS_LEVEL_OUTER:
88 return 0;
89 case VARYING_SLOT_TESS_LEVEL_INNER:
90 return 0x10;
91 default:
92 if (location >= VARYING_SLOT_VAR0 &&
93 location <= VARYING_SLOT_VAR31)
94 return 0x10 * (location - VARYING_SLOT_VAR0) + 0x90;
95
96 if (location >= VARYING_SLOT_PATCH0) {
97 return 0x10 * (location - VARYING_SLOT_PATCH0) + 0x20;
98 }
99 }
100 return 0;
101 }
102
103 static inline nir_ssa_def *
r600_umad_24(nir_builder * b,nir_ssa_def * op1,nir_ssa_def * op2,nir_ssa_def * op3)104 r600_umad_24(nir_builder *b, nir_ssa_def *op1, nir_ssa_def *op2, nir_ssa_def *op3)
105 {
106 return nir_build_alu(b, nir_op_umad24, op1, op2, op3, NULL);
107 }
108
109 static inline nir_ssa_def *
r600_tcs_base_address(nir_builder * b,nir_ssa_def * param_base,nir_ssa_def * rel_patch_id)110 r600_tcs_base_address(nir_builder *b, nir_ssa_def *param_base, nir_ssa_def *rel_patch_id)
111 {
112 return r600_umad_24(b, nir_channel(b, param_base, 0),
113 rel_patch_id,
114 nir_channel(b, param_base, 3));
115 }
116
117
118 static nir_ssa_def *
emil_lsd_in_addr(nir_builder * b,nir_ssa_def * base,nir_ssa_def * patch_id,nir_intrinsic_instr * op)119 emil_lsd_in_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op)
120 {
121 nir_ssa_def *addr = nir_build_alu(b, nir_op_umul24,
122 nir_channel(b, base, 0),
123 patch_id, NULL, NULL);
124
125 auto idx1 = nir_src_as_const_value(op->src[0]);
126 if (!idx1 || idx1->u32 != 0)
127 addr = r600_umad_24(b, nir_channel(b, base, 1),
128 op->src[0].ssa, addr);
129
130 auto offset = nir_imm_int(b, get_tcs_varying_offset(op));
131
132 auto idx2 = nir_src_as_const_value(op->src[1]);
133 if (!idx2 || idx2->u32 != 0)
134 offset = nir_iadd(b, nir_ishl(b, op->src[1].ssa, nir_imm_int(b, 4)), offset);
135
136 return nir_iadd(b, addr, offset);
137 }
138
139 static nir_ssa_def *
emil_lsd_out_addr(nir_builder * b,nir_ssa_def * base,nir_ssa_def * patch_id,nir_intrinsic_instr * op,UNUSED nir_variable_mode mode,int src_offset)140 emil_lsd_out_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op,
141 UNUSED nir_variable_mode mode, int src_offset)
142 {
143
144 nir_ssa_def *addr1 = r600_umad_24(b, nir_channel(b, base, 0),
145 patch_id,
146 nir_channel(b, base, 2));
147 nir_ssa_def *addr2 = r600_umad_24(b, nir_channel(b, base, 1),
148 op->src[src_offset].ssa, addr1);
149 int offset = get_tcs_varying_offset(op);
150 return nir_iadd(b, nir_iadd(b, addr2,
151 nir_ishl(b, op->src[src_offset + 1].ssa, nir_imm_int(b,4))),
152 nir_imm_int(b, offset));
153 }
154
load_offset_group(nir_builder * b,int ncomponents)155 static nir_ssa_def *load_offset_group(nir_builder *b, int ncomponents)
156 {
157 switch (ncomponents) {
158 /* tess outer offsets */
159 case 1: return nir_imm_int(b, 0);
160 case 2: return nir_imm_ivec2(b, 0, 4);
161 case 3: return r600_imm_ivec3(b, 0, 4, 8);
162 case 4: return nir_imm_ivec4(b, 0, 4, 8, 12);
163 /* tess inner offsets */
164 case 5: return nir_imm_int(b, 16);
165 case 6: return nir_imm_ivec2(b, 16, 20);
166 default:
167 debug_printf("Got %d components\n", ncomponents);
168 unreachable("Unsupported component count");
169 }
170 }
171
load_offset_group_from_mask(nir_builder * b,uint32_t mask)172 static nir_ssa_def *load_offset_group_from_mask(nir_builder *b, uint32_t mask)
173 {
174 auto full_mask = nir_imm_ivec4(b, 0, 4, 8, 12);
175 return nir_channels(b, full_mask, mask);
176 }
177
178 struct MaskQuery {
179 uint32_t mask;
180 uint32_t ssa_index;
181 nir_alu_instr *alu;
182 int index;
183 uint32_t full_mask;
184 };
185
update_alu_mask(nir_src * src,void * data)186 static bool update_alu_mask(nir_src *src, void *data)
187 {
188 auto mq = reinterpret_cast<MaskQuery *>(data);
189
190 if (mq->ssa_index == src->ssa->index) {
191 mq->mask |= nir_alu_instr_src_read_mask(mq->alu, mq->index);
192 }
193 ++mq->index;
194
195 return mq->mask != mq->full_mask;
196 }
197
get_dest_usee_mask(nir_intrinsic_instr * op)198 static uint32_t get_dest_usee_mask(nir_intrinsic_instr *op)
199 {
200 assert(op->dest.is_ssa);
201
202 MaskQuery mq = {0};
203 mq.full_mask = (1 << nir_dest_num_components(op->dest)) - 1;
204
205 nir_foreach_use(use_src, &op->dest.ssa) {
206 auto use_instr = use_src->parent_instr;
207 mq.ssa_index = use_src->ssa->index;
208
209 switch (use_instr->type) {
210 case nir_instr_type_alu: {
211 mq.alu = nir_instr_as_alu(use_instr);
212 mq.index = 0;
213 if (!nir_foreach_src(use_instr, update_alu_mask, &mq))
214 return 0xf;
215 break;
216 }
217 case nir_instr_type_intrinsic: {
218 auto intr = nir_instr_as_intrinsic(use_instr);
219 switch (intr->intrinsic) {
220 case nir_intrinsic_store_output:
221 case nir_intrinsic_store_per_vertex_output:
222 mq.mask |= nir_intrinsic_write_mask(intr) << nir_intrinsic_component(intr);
223 break;
224 case nir_intrinsic_store_scratch:
225 case nir_intrinsic_store_local_shared_r600:
226 mq.mask |= nir_intrinsic_write_mask(intr);
227 break;
228 default:
229 return 0xf;
230 }
231 break;
232 }
233 default:
234 return 0xf;
235 }
236
237 }
238 return mq.mask;
239 }
240
replace_load_instr(nir_builder * b,nir_intrinsic_instr * op,nir_ssa_def * addr)241 static void replace_load_instr(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr)
242 {
243 uint32_t mask = get_dest_usee_mask(op);
244 if (mask) {
245 nir_ssa_def *addr_outer = nir_iadd(b, addr, load_offset_group_from_mask(b, mask));
246 if (nir_intrinsic_component(op))
247 addr_outer = nir_iadd(b, addr_outer, nir_imm_int(b, 4 * nir_intrinsic_component(op)));
248
249 auto new_load = nir_load_local_shared_r600(b, 32, addr_outer);
250
251 auto undef = nir_ssa_undef(b, 1, 32);
252 int comps = nir_dest_num_components(op->dest);
253 nir_ssa_def *remix[4] = {undef, undef, undef, undef};
254
255 int chan = 0;
256 for (int i = 0; i < comps; ++i) {
257 if (mask & (1 << i)) {
258 remix[i] = nir_channel(b, new_load, chan++);
259 }
260 }
261 auto new_load_remixed = nir_vec(b, remix, comps);
262 nir_ssa_def_rewrite_uses(&op->dest.ssa, new_load_remixed);
263 }
264 nir_instr_remove(&op->instr);
265 }
266
267 static nir_ssa_def *
r600_load_rel_patch_id(nir_builder * b)268 r600_load_rel_patch_id(nir_builder *b)
269 {
270 auto patch_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_rel_patch_id_r600);
271 nir_ssa_dest_init(&patch_id->instr, &patch_id->dest,
272 1, 32, NULL);
273 nir_builder_instr_insert(b, &patch_id->instr);
274 return &patch_id->dest.ssa;
275 }
276
277 static void
emit_store_lds(nir_builder * b,nir_intrinsic_instr * op,nir_ssa_def * addr)278 emit_store_lds(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr)
279 {
280 uint32_t orig_writemask = nir_intrinsic_write_mask(op) << nir_intrinsic_component(op);
281
282 for (int i = 0; i < 2; ++i) {
283 unsigned test_mask = (0x3 << 2 * i);
284 unsigned wmask = orig_writemask & test_mask;
285 if (!(wmask))
286 continue;
287
288 uint32_t writemask = wmask >> nir_intrinsic_component(op);
289
290 auto store_tcs_out = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_local_shared_r600);
291 nir_intrinsic_set_write_mask(store_tcs_out, writemask);
292 store_tcs_out->src[0] = nir_src_for_ssa(op->src[0].ssa);
293 store_tcs_out->num_components = store_tcs_out->src[0].ssa->num_components;
294 bool start_even = (orig_writemask & (1u << (2 * i)));
295
296 auto addr2 = nir_iadd(b, addr, nir_imm_int(b, 8 * i + (start_even ? 0 : 4)));
297 store_tcs_out->src[1] = nir_src_for_ssa(addr2);
298
299 nir_builder_instr_insert(b, &store_tcs_out->instr);
300 }
301 }
302
303 static nir_ssa_def *
emil_tcs_io_offset(nir_builder * b,nir_ssa_def * addr,nir_intrinsic_instr * op,int src_offset)304 emil_tcs_io_offset(nir_builder *b, nir_ssa_def *addr, nir_intrinsic_instr *op, int src_offset)
305 {
306 int offset = get_tcs_varying_offset(op);
307 return nir_iadd(b, nir_iadd(b, addr,
308 nir_ishl(b, op->src[src_offset].ssa, nir_imm_int(b,4))),
309 nir_imm_int(b, offset));
310 }
311
312
313 inline unsigned
outer_tf_components(pipe_prim_type prim_type)314 outer_tf_components(pipe_prim_type prim_type)
315 {
316 switch (prim_type) {
317 case PIPE_PRIM_LINES: return 2;
318 case PIPE_PRIM_TRIANGLES: return 3;
319 case PIPE_PRIM_QUADS: return 4;
320 default:
321 return 0;
322 }
323 }
324
325
326
327 static bool
r600_lower_tess_io_impl(nir_builder * b,nir_instr * instr,enum pipe_prim_type prim_type)328 r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type prim_type)
329 {
330 static nir_ssa_def *load_in_param_base = nullptr;
331 static nir_ssa_def *load_out_param_base = nullptr;
332
333 b->cursor = nir_before_instr(instr);
334 nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr);
335
336 if (b->shader->info.stage == MESA_SHADER_TESS_CTRL) {
337 load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
338 load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
339 } else if (b->shader->info.stage == MESA_SHADER_TESS_EVAL) {
340 load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
341 } else if (b->shader->info.stage == MESA_SHADER_VERTEX) {
342 load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
343 }
344
345 auto rel_patch_id = r600_load_rel_patch_id(b);
346
347 unsigned tf_inner_address_offset = 0;
348 unsigned ncomps_correct = 0;
349
350 switch (op->intrinsic) {
351 case nir_intrinsic_load_patch_vertices_in: {
352 nir_ssa_def *vertices_in;
353 if (b->shader->info.stage == MESA_SHADER_TESS_CTRL)
354 vertices_in = nir_channel(b, load_in_param_base, 2);
355 else {
356 auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
357 vertices_in = nir_channel(b, base, 2);
358 }
359 nir_ssa_def_rewrite_uses(&op->dest.ssa, vertices_in);
360 nir_instr_remove(&op->instr);
361 return true;
362 }
363 case nir_intrinsic_load_per_vertex_input: {
364 nir_ssa_def *addr =
365 b->shader->info.stage == MESA_SHADER_TESS_CTRL ?
366 emil_lsd_in_addr(b, load_in_param_base, rel_patch_id, op) :
367 emil_lsd_out_addr(b, load_in_param_base, rel_patch_id, op, nir_var_shader_in, 0);
368 replace_load_instr(b, op, addr);
369 return true;
370 }
371 case nir_intrinsic_store_per_vertex_output: {
372 nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 1);
373 emit_store_lds(b, op, addr);
374 nir_instr_remove(instr);
375 return true;
376 }
377 case nir_intrinsic_load_per_vertex_output: {
378 nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 0);
379 replace_load_instr(b, op, addr);
380 return true;
381 }
382 case nir_intrinsic_store_output: {
383 nir_ssa_def *addr = (b->shader->info.stage == MESA_SHADER_TESS_CTRL) ?
384 r600_tcs_base_address(b, load_out_param_base, rel_patch_id):
385 nir_build_alu(b, nir_op_umul24,
386 nir_channel(b, load_out_param_base, 1),
387 rel_patch_id, NULL, NULL);
388 addr = emil_tcs_io_offset(b, addr, op, 1);
389 emit_store_lds(b, op, addr);
390 nir_instr_remove(instr);
391 return true;
392 }
393 case nir_intrinsic_load_output: {
394 nir_ssa_def *addr = r600_tcs_base_address(b, load_out_param_base, rel_patch_id);
395 addr = emil_tcs_io_offset(b, addr, op, 0);
396 replace_load_instr(b, op, addr);
397 return true;
398 }
399 case nir_intrinsic_load_input: {
400 nir_ssa_def *addr = r600_tcs_base_address(b, load_in_param_base, rel_patch_id);
401 addr = emil_tcs_io_offset(b, addr, op, 0);
402 replace_load_instr(b, op, addr);
403 return true;
404 }
405 case nir_intrinsic_load_tess_level_inner:
406 tf_inner_address_offset = 4;
407 ncomps_correct = 2;
408 FALLTHROUGH;
409 case nir_intrinsic_load_tess_level_outer: {
410 auto ncomps = outer_tf_components(prim_type);
411 if (!ncomps)
412 return false;
413 ncomps -= ncomps_correct;
414 auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
415 auto rel_patch_id = r600_load_rel_patch_id(b);
416 nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id);
417 nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, tf_inner_address_offset + ncomps));
418
419 auto tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
420 tf->num_components = ncomps;
421 tf->src[0] = nir_src_for_ssa(addr_outer);
422 nir_ssa_dest_init(&tf->instr, &tf->dest,
423 tf->num_components, 32, NULL);
424 nir_builder_instr_insert(b, &tf->instr);
425
426 nir_ssa_def_rewrite_uses(&op->dest.ssa, &tf->dest.ssa);
427 nir_instr_remove(instr);
428 return true;
429 }
430 default:
431 ;
432 }
433
434 return false;
435 }
436
r600_lower_tess_io(nir_shader * shader,enum pipe_prim_type prim_type)437 bool r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type)
438 {
439 bool progress = false;
440 nir_foreach_function(function, shader) {
441 if (function->impl) {
442 nir_builder b;
443 nir_builder_init(&b, function->impl);
444
445 nir_foreach_block(block, function->impl) {
446 nir_foreach_instr_safe(instr, block) {
447 if (instr->type != nir_instr_type_intrinsic)
448 continue;
449
450 if (r600_lower_tess_io_filter(instr, shader->info.stage))
451 progress |= r600_lower_tess_io_impl(&b, instr, prim_type);
452 }
453 }
454 }
455 }
456 return progress;
457 }
458
r600_emit_tf(nir_builder * b,nir_ssa_def * val)459 bool r600_emit_tf(nir_builder *b, nir_ssa_def *val)
460 {
461 nir_intrinsic_instr *store_tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_tf_r600);
462 store_tf->num_components = val->num_components;
463 store_tf->src[0] = nir_src_for_ssa(val);
464 nir_builder_instr_insert(b, &store_tf->instr);
465 return true;
466 }
467
r600_append_tcs_TF_emission(nir_shader * shader,enum pipe_prim_type prim_type)468 bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type) {
469 if (shader->info.stage != MESA_SHADER_TESS_CTRL)
470 return false;
471
472 nir_foreach_function(function, shader) {
473 nir_foreach_block(block, function->impl) {
474 nir_foreach_instr_safe(instr, block) {
475 if (instr->type != nir_instr_type_intrinsic)
476 continue;
477 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
478 if (intr->intrinsic == nir_intrinsic_store_tf_r600) {
479 return false;
480 }
481 }
482 }
483 }
484 nir_builder builder;
485 nir_builder *b = &builder;
486
487 assert(exec_list_length(&shader->functions) == 1);
488 nir_function *f = (nir_function *)shader->functions.get_head();
489 nir_builder_init(b, f->impl);
490
491 auto outer_comps = outer_tf_components(prim_type);
492 if (!outer_comps)
493 return false;
494
495 unsigned inner_comps = outer_comps - 2;
496 unsigned stride = (inner_comps + outer_comps) * 4;
497
498 b->cursor = nir_after_cf_list(&f->impl->body);
499
500 auto invocation_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_invocation_id);
501 nir_ssa_dest_init(&invocation_id->instr, &invocation_id->dest,
502 1, 32, NULL);
503 nir_builder_instr_insert(b, &invocation_id->instr);
504
505 nir_push_if(b, nir_ieq_imm(b, &invocation_id->dest.ssa, 0));
506 auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
507 auto rel_patch_id = r600_load_rel_patch_id(b);
508
509 nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id);
510
511 nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, outer_comps));
512 auto tf_outer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
513 tf_outer->num_components = outer_comps;
514 tf_outer->src[0] = nir_src_for_ssa(addr_outer);
515 nir_ssa_dest_init(&tf_outer->instr, &tf_outer->dest,
516 tf_outer->num_components, 32, NULL);
517 nir_builder_instr_insert(b, &tf_outer->instr);
518
519 std::vector<nir_ssa_def *> tf_out;
520
521
522 auto tf_out_base = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_tess_factor_base_r600);
523 nir_ssa_dest_init(&tf_out_base->instr, &tf_out_base->dest,
524 1, 32, NULL);
525 nir_builder_instr_insert(b, &tf_out_base->instr);
526
527 auto out_addr0 = nir_build_alu(b, nir_op_umad24,
528 rel_patch_id,
529 nir_imm_int(b, stride),
530 &tf_out_base->dest.ssa,
531 NULL);
532 int chanx = 0;
533 int chany = 1;
534
535 if (prim_type == PIPE_PRIM_LINES)
536 std::swap(chanx, chany);
537
538
539 auto v0 = nir_vec4(b, out_addr0, nir_channel(b, &tf_outer->dest.ssa, chanx),
540 nir_iadd(b, out_addr0, nir_imm_int(b, 4)),
541 nir_channel(b, &tf_outer->dest.ssa, chany));
542
543 tf_out.push_back(v0);
544 if (outer_comps > 2) {
545 auto v1 = (outer_comps > 3) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)),
546 nir_channel(b, &tf_outer->dest.ssa, 2),
547 nir_iadd(b, out_addr0, nir_imm_int(b, 12)),
548 nir_channel(b, &tf_outer->dest.ssa, 3)) :
549 nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)),
550 nir_channel(b, &tf_outer->dest.ssa, 2));
551 tf_out.push_back(v1);
552 }
553
554 if (inner_comps) {
555 nir_ssa_def *addr1 = nir_iadd(b, addr0, load_offset_group(b, 4 + inner_comps));
556 auto tf_inner = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
557 tf_inner->num_components = inner_comps;
558 tf_inner->src[0] = nir_src_for_ssa(addr1);
559 nir_ssa_dest_init(&tf_inner->instr, &tf_inner->dest,
560 tf_inner->num_components, 32, NULL);
561 nir_builder_instr_insert(b, &tf_inner->instr);
562
563 auto v2 = (inner_comps > 1) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 16)),
564 nir_channel(b, &tf_inner->dest.ssa, 0),
565 nir_iadd(b, out_addr0, nir_imm_int(b, 20)),
566 nir_channel(b, &tf_inner->dest.ssa, 1)):
567 nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 12)),
568 nir_channel(b, &tf_inner->dest.ssa, 0));
569 tf_out.push_back(v2);
570 }
571
572 for (auto tf: tf_out)
573 r600_emit_tf(b, tf);
574
575 nir_pop_if(b, nullptr);
576
577 nir_metadata_preserve(f->impl, nir_metadata_none);
578
579 return true;
580 }
581
582 static bool
r600_lower_tess_coord_filter(const nir_instr * instr,UNUSED const void * _options)583 r600_lower_tess_coord_filter(const nir_instr *instr, UNUSED const void *_options)
584 {
585 if (instr->type != nir_instr_type_intrinsic)
586 return false;
587 auto intr = nir_instr_as_intrinsic(instr);
588 return intr->intrinsic == nir_intrinsic_load_tess_coord;
589 }
590
591 static nir_ssa_def *
r600_lower_tess_coord_impl(nir_builder * b,UNUSED nir_instr * instr,void * _options)592 r600_lower_tess_coord_impl(nir_builder *b, UNUSED nir_instr *instr, void *_options)
593 {
594 pipe_prim_type prim_type = *(pipe_prim_type *)_options;
595
596 auto tc_xy = nir_load_tess_coord_r600(b);
597
598 auto tc_x = nir_channel(b, tc_xy, 0);
599 auto tc_y = nir_channel(b, tc_xy, 1);
600
601 if (prim_type == PIPE_PRIM_TRIANGLES)
602 return nir_vec3(b, tc_x, tc_y, nir_fsub(b, nir_imm_float(b, 1.0),
603 nir_fadd(b, tc_x, tc_y)));
604 else
605 return nir_vec3(b, tc_x, tc_y, nir_imm_float(b, 0.0));
606 }
607
608
r600_lower_tess_coord(nir_shader * sh,enum pipe_prim_type prim_type)609 bool r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type)
610 {
611 return nir_shader_lower_instructions(sh, r600_lower_tess_coord_filter,
612 r600_lower_tess_coord_impl, &prim_type);
613 }
614