1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "tgsi/tgsi_build.h"
24 #include "tgsi/tgsi_dump.h"
25 #include "tgsi/tgsi_scan.h"
26 #include "tgsi/tgsi_util.h"
27
28 #include <set>
29
30 #include "codegen/nv50_ir.h"
31 #include "codegen/nv50_ir_from_common.h"
32 #include "codegen/nv50_ir_util.h"
33
34 namespace tgsi {
35
36 class Source;
37
38 static nv50_ir::operation translateOpcode(uint opcode);
39 static nv50_ir::DataFile translateFile(uint file);
40 static nv50_ir::TexTarget translateTexture(uint texTarg);
41 static nv50_ir::SVSemantic translateSysVal(uint sysval);
42 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
43
44 class Instruction
45 {
46 public:
Instruction(const struct tgsi_full_instruction * inst)47 Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
48
49 class SrcRegister
50 {
51 public:
SrcRegister(const struct tgsi_full_src_register * src)52 SrcRegister(const struct tgsi_full_src_register *src)
53 : reg(src->Register),
54 fsr(src)
55 { }
56
SrcRegister(const struct tgsi_src_register & src)57 SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
58
SrcRegister(const struct tgsi_ind_register & ind)59 SrcRegister(const struct tgsi_ind_register& ind)
60 : reg(tgsi_util_get_src_from_ind(&ind)),
61 fsr(NULL)
62 { }
63
offsetToSrc(struct tgsi_texture_offset off)64 struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
65 {
66 struct tgsi_src_register reg;
67 memset(®, 0, sizeof(reg));
68 reg.Index = off.Index;
69 reg.File = off.File;
70 reg.SwizzleX = off.SwizzleX;
71 reg.SwizzleY = off.SwizzleY;
72 reg.SwizzleZ = off.SwizzleZ;
73 return reg;
74 }
75
SrcRegister(const struct tgsi_texture_offset & off)76 SrcRegister(const struct tgsi_texture_offset& off) :
77 reg(offsetToSrc(off)),
78 fsr(NULL)
79 { }
80
getFile() const81 uint getFile() const { return reg.File; }
82
is2D() const83 bool is2D() const { return reg.Dimension; }
84
isIndirect(int dim) const85 bool isIndirect(int dim) const
86 {
87 return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
88 }
89
getIndex(int dim) const90 int getIndex(int dim) const
91 {
92 return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
93 }
94
getSwizzle(int chan) const95 int getSwizzle(int chan) const
96 {
97 return tgsi_util_get_src_register_swizzle(®, chan);
98 }
99
getArrayId() const100 int getArrayId() const
101 {
102 if (isIndirect(0))
103 return fsr->Indirect.ArrayID;
104 return 0;
105 }
106
107 nv50_ir::Modifier getMod(int chan) const;
108
getIndirect(int dim) const109 SrcRegister getIndirect(int dim) const
110 {
111 assert(fsr && isIndirect(dim));
112 if (dim)
113 return SrcRegister(fsr->DimIndirect);
114 return SrcRegister(fsr->Indirect);
115 }
116
getValueU32(int c,const uint32_t * data) const117 uint32_t getValueU32(int c, const uint32_t *data) const
118 {
119 assert(reg.File == TGSI_FILE_IMMEDIATE);
120 assert(!reg.Absolute);
121 assert(!reg.Negate);
122 return data[reg.Index * 4 + getSwizzle(c)];
123 }
124
125 private:
126 const struct tgsi_src_register reg;
127 const struct tgsi_full_src_register *fsr;
128 };
129
130 class DstRegister
131 {
132 public:
DstRegister(const struct tgsi_full_dst_register * dst)133 DstRegister(const struct tgsi_full_dst_register *dst)
134 : reg(dst->Register),
135 fdr(dst)
136 { }
137
DstRegister(const struct tgsi_dst_register & dst)138 DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
139
getFile() const140 uint getFile() const { return reg.File; }
141
is2D() const142 bool is2D() const { return reg.Dimension; }
143
isIndirect(int dim) const144 bool isIndirect(int dim) const
145 {
146 return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
147 }
148
getIndex(int dim) const149 int getIndex(int dim) const
150 {
151 return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
152 }
153
getMask() const154 unsigned int getMask() const { return reg.WriteMask; }
155
isMasked(int chan) const156 bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
157
getIndirect(int dim) const158 SrcRegister getIndirect(int dim) const
159 {
160 assert(fdr && isIndirect(dim));
161 if (dim)
162 return SrcRegister(fdr->DimIndirect);
163 return SrcRegister(fdr->Indirect);
164 }
165
asSrc()166 struct tgsi_full_src_register asSrc()
167 {
168 assert(fdr);
169 return tgsi_full_src_register_from_dst(fdr);
170 }
171
getArrayId() const172 int getArrayId() const
173 {
174 if (isIndirect(0))
175 return fdr->Indirect.ArrayID;
176 return 0;
177 }
178
179 private:
180 const struct tgsi_dst_register reg;
181 const struct tgsi_full_dst_register *fdr;
182 };
183
getOpcode() const184 inline uint getOpcode() const { return insn->Instruction.Opcode; }
185
srcCount() const186 unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
dstCount() const187 unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
188
189 // mask of used components of source s
190 unsigned int srcMask(unsigned int s) const;
191 unsigned int texOffsetMask() const;
192
getSrc(unsigned int s) const193 SrcRegister getSrc(unsigned int s) const
194 {
195 assert(s < srcCount());
196 return SrcRegister(&insn->Src[s]);
197 }
198
getDst(unsigned int d) const199 DstRegister getDst(unsigned int d) const
200 {
201 assert(d < dstCount());
202 return DstRegister(&insn->Dst[d]);
203 }
204
getTexOffset(unsigned int i) const205 SrcRegister getTexOffset(unsigned int i) const
206 {
207 assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
208 return SrcRegister(insn->TexOffsets[i]);
209 }
210
getNumTexOffsets() const211 unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
212
213 bool checkDstSrcAliasing() const;
214
getOP() const215 inline nv50_ir::operation getOP() const {
216 return translateOpcode(getOpcode()); }
217
218 nv50_ir::DataType inferSrcType() const;
219 nv50_ir::DataType inferDstType() const;
220
221 nv50_ir::CondCode getSetCond() const;
222
223 nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
224
getImageFormat() const225 const nv50_ir::TexInstruction::ImgFormatDesc *getImageFormat() const {
226 return nv50_ir::TexInstruction::translateImgFormat((enum pipe_format)insn->Memory.Format);
227 }
228
getImageTarget() const229 nv50_ir::TexTarget getImageTarget() const {
230 return translateTexture(insn->Memory.Texture);
231 }
232
getCacheMode() const233 nv50_ir::CacheMode getCacheMode() const {
234 if (!insn->Instruction.Memory)
235 return nv50_ir::CACHE_CA;
236 return translateCacheMode(insn->Memory.Qualifier);
237 }
238
getLabel()239 inline uint getLabel() { return insn->Label.Label; }
240
getSaturate() const241 unsigned getSaturate() const { return insn->Instruction.Saturate; }
242
print() const243 void print() const
244 {
245 tgsi_dump_instruction(insn, 1);
246 }
247
248 private:
249 const struct tgsi_full_instruction *insn;
250 };
251
texOffsetMask() const252 unsigned int Instruction::texOffsetMask() const
253 {
254 const struct tgsi_instruction_texture *tex = &insn->Texture;
255 assert(insn->Instruction.Texture);
256
257 switch (tex->Texture) {
258 case TGSI_TEXTURE_BUFFER:
259 case TGSI_TEXTURE_1D:
260 case TGSI_TEXTURE_SHADOW1D:
261 case TGSI_TEXTURE_1D_ARRAY:
262 case TGSI_TEXTURE_SHADOW1D_ARRAY:
263 return 0x1;
264 case TGSI_TEXTURE_2D:
265 case TGSI_TEXTURE_SHADOW2D:
266 case TGSI_TEXTURE_2D_ARRAY:
267 case TGSI_TEXTURE_SHADOW2D_ARRAY:
268 case TGSI_TEXTURE_RECT:
269 case TGSI_TEXTURE_SHADOWRECT:
270 case TGSI_TEXTURE_2D_MSAA:
271 case TGSI_TEXTURE_2D_ARRAY_MSAA:
272 return 0x3;
273 case TGSI_TEXTURE_3D:
274 return 0x7;
275 default:
276 assert(!"Unexpected texture target");
277 return 0xf;
278 }
279 }
280
srcMask(unsigned int s) const281 unsigned int Instruction::srcMask(unsigned int s) const
282 {
283 unsigned int mask = insn->Dst[0].Register.WriteMask;
284
285 switch (insn->Instruction.Opcode) {
286 case TGSI_OPCODE_COS:
287 case TGSI_OPCODE_SIN:
288 return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
289 case TGSI_OPCODE_DP2:
290 return 0x3;
291 case TGSI_OPCODE_DP3:
292 return 0x7;
293 case TGSI_OPCODE_DP4:
294 case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
295 return 0xf;
296 case TGSI_OPCODE_DST:
297 return mask & (s ? 0xa : 0x6);
298 case TGSI_OPCODE_EX2:
299 case TGSI_OPCODE_EXP:
300 case TGSI_OPCODE_LG2:
301 case TGSI_OPCODE_LOG:
302 case TGSI_OPCODE_POW:
303 case TGSI_OPCODE_RCP:
304 case TGSI_OPCODE_RSQ:
305 return 0x1;
306 case TGSI_OPCODE_IF:
307 case TGSI_OPCODE_UIF:
308 return 0x1;
309 case TGSI_OPCODE_LIT:
310 return 0xb;
311 case TGSI_OPCODE_TEX2:
312 case TGSI_OPCODE_TXB2:
313 case TGSI_OPCODE_TXL2:
314 return (s == 0) ? 0xf : 0x3;
315 case TGSI_OPCODE_TEX:
316 case TGSI_OPCODE_TXB:
317 case TGSI_OPCODE_TXD:
318 case TGSI_OPCODE_TXL:
319 case TGSI_OPCODE_TXP:
320 case TGSI_OPCODE_TXF:
321 case TGSI_OPCODE_TG4:
322 case TGSI_OPCODE_TEX_LZ:
323 case TGSI_OPCODE_TXF_LZ:
324 case TGSI_OPCODE_LODQ:
325 {
326 const struct tgsi_instruction_texture *tex = &insn->Texture;
327
328 assert(insn->Instruction.Texture);
329
330 mask = 0x7;
331 if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
332 insn->Instruction.Opcode != TGSI_OPCODE_TEX_LZ &&
333 insn->Instruction.Opcode != TGSI_OPCODE_TXF_LZ &&
334 insn->Instruction.Opcode != TGSI_OPCODE_TXD)
335 mask |= 0x8; /* bias, lod or proj */
336
337 switch (tex->Texture) {
338 case TGSI_TEXTURE_1D:
339 mask &= 0x9;
340 break;
341 case TGSI_TEXTURE_SHADOW1D:
342 mask &= 0xd;
343 break;
344 case TGSI_TEXTURE_1D_ARRAY:
345 case TGSI_TEXTURE_2D:
346 case TGSI_TEXTURE_RECT:
347 mask &= 0xb;
348 break;
349 case TGSI_TEXTURE_CUBE_ARRAY:
350 case TGSI_TEXTURE_SHADOW2D_ARRAY:
351 case TGSI_TEXTURE_SHADOWCUBE:
352 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
353 mask |= 0x8;
354 break;
355 default:
356 break;
357 }
358 }
359 return mask;
360 case TGSI_OPCODE_TXQ:
361 return 1;
362 case TGSI_OPCODE_D2I:
363 case TGSI_OPCODE_D2U:
364 case TGSI_OPCODE_D2F:
365 case TGSI_OPCODE_DSLT:
366 case TGSI_OPCODE_DSGE:
367 case TGSI_OPCODE_DSEQ:
368 case TGSI_OPCODE_DSNE:
369 case TGSI_OPCODE_U64SEQ:
370 case TGSI_OPCODE_U64SNE:
371 case TGSI_OPCODE_I64SLT:
372 case TGSI_OPCODE_U64SLT:
373 case TGSI_OPCODE_I64SGE:
374 case TGSI_OPCODE_U64SGE:
375 case TGSI_OPCODE_I642F:
376 case TGSI_OPCODE_U642F:
377 switch (util_bitcount(mask)) {
378 case 1: return 0x3;
379 case 2: return 0xf;
380 default:
381 assert(!"unexpected mask");
382 return 0xf;
383 }
384 case TGSI_OPCODE_I2D:
385 case TGSI_OPCODE_U2D:
386 case TGSI_OPCODE_F2D: {
387 unsigned int x = 0;
388 if ((mask & 0x3) == 0x3)
389 x |= 1;
390 if ((mask & 0xc) == 0xc)
391 x |= 2;
392 return x;
393 }
394 case TGSI_OPCODE_PK2H:
395 return 0x3;
396 case TGSI_OPCODE_UP2H:
397 return 0x1;
398 default:
399 break;
400 }
401
402 return mask;
403 }
404
getMod(int chan) const405 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
406 {
407 nv50_ir::Modifier m(0);
408
409 if (reg.Absolute)
410 m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
411 if (reg.Negate)
412 m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
413 return m;
414 }
415
translateFile(uint file)416 static nv50_ir::DataFile translateFile(uint file)
417 {
418 switch (file) {
419 case TGSI_FILE_CONSTANT: return nv50_ir::FILE_MEMORY_CONST;
420 case TGSI_FILE_INPUT: return nv50_ir::FILE_SHADER_INPUT;
421 case TGSI_FILE_OUTPUT: return nv50_ir::FILE_SHADER_OUTPUT;
422 case TGSI_FILE_TEMPORARY: return nv50_ir::FILE_GPR;
423 case TGSI_FILE_ADDRESS: return nv50_ir::FILE_ADDRESS;
424 case TGSI_FILE_IMMEDIATE: return nv50_ir::FILE_IMMEDIATE;
425 case TGSI_FILE_SYSTEM_VALUE: return nv50_ir::FILE_SYSTEM_VALUE;
426 case TGSI_FILE_BUFFER: return nv50_ir::FILE_MEMORY_BUFFER;
427 case TGSI_FILE_IMAGE: return nv50_ir::FILE_MEMORY_GLOBAL;
428 case TGSI_FILE_MEMORY: return nv50_ir::FILE_MEMORY_GLOBAL;
429 case TGSI_FILE_SAMPLER:
430 case TGSI_FILE_NULL:
431 default:
432 return nv50_ir::FILE_NULL;
433 }
434 }
435
translateSysVal(uint sysval)436 static nv50_ir::SVSemantic translateSysVal(uint sysval)
437 {
438 switch (sysval) {
439 case TGSI_SEMANTIC_FACE: return nv50_ir::SV_FACE;
440 case TGSI_SEMANTIC_PSIZE: return nv50_ir::SV_POINT_SIZE;
441 case TGSI_SEMANTIC_PRIMID: return nv50_ir::SV_PRIMITIVE_ID;
442 case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
443 case TGSI_SEMANTIC_VERTEXID: return nv50_ir::SV_VERTEX_ID;
444 case TGSI_SEMANTIC_GRID_SIZE: return nv50_ir::SV_NCTAID;
445 case TGSI_SEMANTIC_BLOCK_ID: return nv50_ir::SV_CTAID;
446 case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
447 case TGSI_SEMANTIC_THREAD_ID: return nv50_ir::SV_TID;
448 case TGSI_SEMANTIC_SAMPLEID: return nv50_ir::SV_SAMPLE_INDEX;
449 case TGSI_SEMANTIC_SAMPLEPOS: return nv50_ir::SV_SAMPLE_POS;
450 case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
451 case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
452 case TGSI_SEMANTIC_TESSCOORD: return nv50_ir::SV_TESS_COORD;
453 case TGSI_SEMANTIC_TESSOUTER: return nv50_ir::SV_TESS_OUTER;
454 case TGSI_SEMANTIC_TESSINNER: return nv50_ir::SV_TESS_INNER;
455 case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
456 case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
457 case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
458 case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
459 case TGSI_SEMANTIC_DRAWID: return nv50_ir::SV_DRAWID;
460 case TGSI_SEMANTIC_WORK_DIM: return nv50_ir::SV_WORK_DIM;
461 case TGSI_SEMANTIC_SUBGROUP_INVOCATION: return nv50_ir::SV_LANEID;
462 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK: return nv50_ir::SV_LANEMASK_EQ;
463 case TGSI_SEMANTIC_SUBGROUP_LT_MASK: return nv50_ir::SV_LANEMASK_LT;
464 case TGSI_SEMANTIC_SUBGROUP_LE_MASK: return nv50_ir::SV_LANEMASK_LE;
465 case TGSI_SEMANTIC_SUBGROUP_GT_MASK: return nv50_ir::SV_LANEMASK_GT;
466 case TGSI_SEMANTIC_SUBGROUP_GE_MASK: return nv50_ir::SV_LANEMASK_GE;
467 default:
468 assert(0);
469 return nv50_ir::SV_CLOCK;
470 }
471 }
472
473 #define NV50_IR_TEX_TARG_CASE(a, b) \
474 case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
475
translateTexture(uint tex)476 static nv50_ir::TexTarget translateTexture(uint tex)
477 {
478 switch (tex) {
479 NV50_IR_TEX_TARG_CASE(1D, 1D);
480 NV50_IR_TEX_TARG_CASE(2D, 2D);
481 NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
482 NV50_IR_TEX_TARG_CASE(3D, 3D);
483 NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
484 NV50_IR_TEX_TARG_CASE(RECT, RECT);
485 NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
486 NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
487 NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
488 NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
489 NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
490 NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
491 NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
492 NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
493 NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
494 NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
495 NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
496 NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
497
498 case TGSI_TEXTURE_UNKNOWN:
499 default:
500 assert(!"invalid texture target");
501 return nv50_ir::TEX_TARGET_2D;
502 }
503 }
504
translateCacheMode(uint qualifier)505 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
506 {
507 if (qualifier & TGSI_MEMORY_VOLATILE)
508 return nv50_ir::CACHE_CV;
509 if (qualifier & TGSI_MEMORY_COHERENT)
510 return nv50_ir::CACHE_CG;
511 return nv50_ir::CACHE_CA;
512 }
513
inferSrcType() const514 nv50_ir::DataType Instruction::inferSrcType() const
515 {
516 switch (getOpcode()) {
517 case TGSI_OPCODE_UIF:
518 case TGSI_OPCODE_AND:
519 case TGSI_OPCODE_OR:
520 case TGSI_OPCODE_XOR:
521 case TGSI_OPCODE_NOT:
522 case TGSI_OPCODE_SHL:
523 case TGSI_OPCODE_U2F:
524 case TGSI_OPCODE_U2D:
525 case TGSI_OPCODE_U2I64:
526 case TGSI_OPCODE_UADD:
527 case TGSI_OPCODE_UDIV:
528 case TGSI_OPCODE_UMOD:
529 case TGSI_OPCODE_UMAD:
530 case TGSI_OPCODE_UMUL:
531 case TGSI_OPCODE_UMUL_HI:
532 case TGSI_OPCODE_UMAX:
533 case TGSI_OPCODE_UMIN:
534 case TGSI_OPCODE_USEQ:
535 case TGSI_OPCODE_USGE:
536 case TGSI_OPCODE_USLT:
537 case TGSI_OPCODE_USNE:
538 case TGSI_OPCODE_USHR:
539 case TGSI_OPCODE_ATOMUADD:
540 case TGSI_OPCODE_ATOMXCHG:
541 case TGSI_OPCODE_ATOMCAS:
542 case TGSI_OPCODE_ATOMAND:
543 case TGSI_OPCODE_ATOMOR:
544 case TGSI_OPCODE_ATOMXOR:
545 case TGSI_OPCODE_ATOMUMIN:
546 case TGSI_OPCODE_ATOMUMAX:
547 case TGSI_OPCODE_ATOMDEC_WRAP:
548 case TGSI_OPCODE_ATOMINC_WRAP:
549 case TGSI_OPCODE_UBFE:
550 case TGSI_OPCODE_UMSB:
551 case TGSI_OPCODE_UP2H:
552 case TGSI_OPCODE_VOTE_ALL:
553 case TGSI_OPCODE_VOTE_ANY:
554 case TGSI_OPCODE_VOTE_EQ:
555 return nv50_ir::TYPE_U32;
556 case TGSI_OPCODE_I2F:
557 case TGSI_OPCODE_I2D:
558 case TGSI_OPCODE_I2I64:
559 case TGSI_OPCODE_IDIV:
560 case TGSI_OPCODE_IMUL_HI:
561 case TGSI_OPCODE_IMAX:
562 case TGSI_OPCODE_IMIN:
563 case TGSI_OPCODE_IABS:
564 case TGSI_OPCODE_INEG:
565 case TGSI_OPCODE_ISGE:
566 case TGSI_OPCODE_ISHR:
567 case TGSI_OPCODE_ISLT:
568 case TGSI_OPCODE_ISSG:
569 case TGSI_OPCODE_MOD:
570 case TGSI_OPCODE_UARL:
571 case TGSI_OPCODE_ATOMIMIN:
572 case TGSI_OPCODE_ATOMIMAX:
573 case TGSI_OPCODE_IBFE:
574 case TGSI_OPCODE_IMSB:
575 return nv50_ir::TYPE_S32;
576 case TGSI_OPCODE_D2F:
577 case TGSI_OPCODE_D2I:
578 case TGSI_OPCODE_D2U:
579 case TGSI_OPCODE_D2I64:
580 case TGSI_OPCODE_D2U64:
581 case TGSI_OPCODE_DABS:
582 case TGSI_OPCODE_DNEG:
583 case TGSI_OPCODE_DADD:
584 case TGSI_OPCODE_DMUL:
585 case TGSI_OPCODE_DDIV:
586 case TGSI_OPCODE_DMAX:
587 case TGSI_OPCODE_DMIN:
588 case TGSI_OPCODE_DSLT:
589 case TGSI_OPCODE_DSGE:
590 case TGSI_OPCODE_DSEQ:
591 case TGSI_OPCODE_DSNE:
592 case TGSI_OPCODE_DRCP:
593 case TGSI_OPCODE_DSQRT:
594 case TGSI_OPCODE_DMAD:
595 case TGSI_OPCODE_DFMA:
596 case TGSI_OPCODE_DFRAC:
597 case TGSI_OPCODE_DRSQ:
598 case TGSI_OPCODE_DTRUNC:
599 case TGSI_OPCODE_DCEIL:
600 case TGSI_OPCODE_DFLR:
601 case TGSI_OPCODE_DROUND:
602 return nv50_ir::TYPE_F64;
603 case TGSI_OPCODE_U64SEQ:
604 case TGSI_OPCODE_U64SNE:
605 case TGSI_OPCODE_U64SLT:
606 case TGSI_OPCODE_U64SGE:
607 case TGSI_OPCODE_U64MIN:
608 case TGSI_OPCODE_U64MAX:
609 case TGSI_OPCODE_U64ADD:
610 case TGSI_OPCODE_U64MUL:
611 case TGSI_OPCODE_U64SHL:
612 case TGSI_OPCODE_U64SHR:
613 case TGSI_OPCODE_U64DIV:
614 case TGSI_OPCODE_U64MOD:
615 case TGSI_OPCODE_U642F:
616 case TGSI_OPCODE_U642D:
617 return nv50_ir::TYPE_U64;
618 case TGSI_OPCODE_I64ABS:
619 case TGSI_OPCODE_I64SSG:
620 case TGSI_OPCODE_I64NEG:
621 case TGSI_OPCODE_I64SLT:
622 case TGSI_OPCODE_I64SGE:
623 case TGSI_OPCODE_I64MIN:
624 case TGSI_OPCODE_I64MAX:
625 case TGSI_OPCODE_I64SHR:
626 case TGSI_OPCODE_I64DIV:
627 case TGSI_OPCODE_I64MOD:
628 case TGSI_OPCODE_I642F:
629 case TGSI_OPCODE_I642D:
630 return nv50_ir::TYPE_S64;
631 default:
632 return nv50_ir::TYPE_F32;
633 }
634 }
635
inferDstType() const636 nv50_ir::DataType Instruction::inferDstType() const
637 {
638 switch (getOpcode()) {
639 case TGSI_OPCODE_D2U:
640 case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
641 case TGSI_OPCODE_D2I:
642 case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
643 case TGSI_OPCODE_FSEQ:
644 case TGSI_OPCODE_FSGE:
645 case TGSI_OPCODE_FSLT:
646 case TGSI_OPCODE_FSNE:
647 case TGSI_OPCODE_DSEQ:
648 case TGSI_OPCODE_DSGE:
649 case TGSI_OPCODE_DSLT:
650 case TGSI_OPCODE_DSNE:
651 case TGSI_OPCODE_I64SLT:
652 case TGSI_OPCODE_I64SGE:
653 case TGSI_OPCODE_U64SEQ:
654 case TGSI_OPCODE_U64SNE:
655 case TGSI_OPCODE_U64SLT:
656 case TGSI_OPCODE_U64SGE:
657 case TGSI_OPCODE_PK2H:
658 return nv50_ir::TYPE_U32;
659 case TGSI_OPCODE_I2F:
660 case TGSI_OPCODE_U2F:
661 case TGSI_OPCODE_D2F:
662 case TGSI_OPCODE_I642F:
663 case TGSI_OPCODE_U642F:
664 case TGSI_OPCODE_UP2H:
665 return nv50_ir::TYPE_F32;
666 case TGSI_OPCODE_I2D:
667 case TGSI_OPCODE_U2D:
668 case TGSI_OPCODE_F2D:
669 case TGSI_OPCODE_I642D:
670 case TGSI_OPCODE_U642D:
671 return nv50_ir::TYPE_F64;
672 case TGSI_OPCODE_I2I64:
673 case TGSI_OPCODE_U2I64:
674 case TGSI_OPCODE_F2I64:
675 case TGSI_OPCODE_D2I64:
676 return nv50_ir::TYPE_S64;
677 case TGSI_OPCODE_F2U64:
678 case TGSI_OPCODE_D2U64:
679 return nv50_ir::TYPE_U64;
680 default:
681 return inferSrcType();
682 }
683 }
684
getSetCond() const685 nv50_ir::CondCode Instruction::getSetCond() const
686 {
687 using namespace nv50_ir;
688
689 switch (getOpcode()) {
690 case TGSI_OPCODE_SLT:
691 case TGSI_OPCODE_ISLT:
692 case TGSI_OPCODE_USLT:
693 case TGSI_OPCODE_FSLT:
694 case TGSI_OPCODE_DSLT:
695 case TGSI_OPCODE_I64SLT:
696 case TGSI_OPCODE_U64SLT:
697 return CC_LT;
698 case TGSI_OPCODE_SLE:
699 return CC_LE;
700 case TGSI_OPCODE_SGE:
701 case TGSI_OPCODE_ISGE:
702 case TGSI_OPCODE_USGE:
703 case TGSI_OPCODE_FSGE:
704 case TGSI_OPCODE_DSGE:
705 case TGSI_OPCODE_I64SGE:
706 case TGSI_OPCODE_U64SGE:
707 return CC_GE;
708 case TGSI_OPCODE_SGT:
709 return CC_GT;
710 case TGSI_OPCODE_SEQ:
711 case TGSI_OPCODE_USEQ:
712 case TGSI_OPCODE_FSEQ:
713 case TGSI_OPCODE_DSEQ:
714 case TGSI_OPCODE_U64SEQ:
715 return CC_EQ;
716 case TGSI_OPCODE_SNE:
717 case TGSI_OPCODE_FSNE:
718 case TGSI_OPCODE_DSNE:
719 case TGSI_OPCODE_U64SNE:
720 return CC_NEU;
721 case TGSI_OPCODE_USNE:
722 return CC_NE;
723 default:
724 return CC_ALWAYS;
725 }
726 }
727
728 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
729
translateOpcode(uint opcode)730 static nv50_ir::operation translateOpcode(uint opcode)
731 {
732 switch (opcode) {
733 NV50_IR_OPCODE_CASE(ARL, SHL);
734 NV50_IR_OPCODE_CASE(MOV, MOV);
735
736 NV50_IR_OPCODE_CASE(RCP, RCP);
737 NV50_IR_OPCODE_CASE(RSQ, RSQ);
738 NV50_IR_OPCODE_CASE(SQRT, SQRT);
739
740 NV50_IR_OPCODE_CASE(MUL, MUL);
741 NV50_IR_OPCODE_CASE(ADD, ADD);
742
743 NV50_IR_OPCODE_CASE(MIN, MIN);
744 NV50_IR_OPCODE_CASE(MAX, MAX);
745 NV50_IR_OPCODE_CASE(SLT, SET);
746 NV50_IR_OPCODE_CASE(SGE, SET);
747 NV50_IR_OPCODE_CASE(MAD, MAD);
748 NV50_IR_OPCODE_CASE(FMA, FMA);
749
750 NV50_IR_OPCODE_CASE(FLR, FLOOR);
751 NV50_IR_OPCODE_CASE(ROUND, CVT);
752 NV50_IR_OPCODE_CASE(EX2, EX2);
753 NV50_IR_OPCODE_CASE(LG2, LG2);
754 NV50_IR_OPCODE_CASE(POW, POW);
755
756 NV50_IR_OPCODE_CASE(COS, COS);
757 NV50_IR_OPCODE_CASE(DDX, DFDX);
758 NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
759 NV50_IR_OPCODE_CASE(DDY, DFDY);
760 NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
761 NV50_IR_OPCODE_CASE(KILL, DISCARD);
762 NV50_IR_OPCODE_CASE(DEMOTE, DISCARD);
763
764 NV50_IR_OPCODE_CASE(SEQ, SET);
765 NV50_IR_OPCODE_CASE(SGT, SET);
766 NV50_IR_OPCODE_CASE(SIN, SIN);
767 NV50_IR_OPCODE_CASE(SLE, SET);
768 NV50_IR_OPCODE_CASE(SNE, SET);
769 NV50_IR_OPCODE_CASE(TEX, TEX);
770 NV50_IR_OPCODE_CASE(TXD, TXD);
771 NV50_IR_OPCODE_CASE(TXP, TEX);
772
773 NV50_IR_OPCODE_CASE(CAL, CALL);
774 NV50_IR_OPCODE_CASE(RET, RET);
775 NV50_IR_OPCODE_CASE(CMP, SLCT);
776
777 NV50_IR_OPCODE_CASE(TXB, TXB);
778
779 NV50_IR_OPCODE_CASE(DIV, DIV);
780
781 NV50_IR_OPCODE_CASE(TXL, TXL);
782 NV50_IR_OPCODE_CASE(TEX_LZ, TXL);
783
784 NV50_IR_OPCODE_CASE(CEIL, CEIL);
785 NV50_IR_OPCODE_CASE(I2F, CVT);
786 NV50_IR_OPCODE_CASE(NOT, NOT);
787 NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
788 NV50_IR_OPCODE_CASE(SHL, SHL);
789
790 NV50_IR_OPCODE_CASE(AND, AND);
791 NV50_IR_OPCODE_CASE(OR, OR);
792 NV50_IR_OPCODE_CASE(MOD, MOD);
793 NV50_IR_OPCODE_CASE(XOR, XOR);
794 NV50_IR_OPCODE_CASE(TXF, TXF);
795 NV50_IR_OPCODE_CASE(TXF_LZ, TXF);
796 NV50_IR_OPCODE_CASE(TXQ, TXQ);
797 NV50_IR_OPCODE_CASE(TXQS, TXQ);
798 NV50_IR_OPCODE_CASE(TG4, TXG);
799 NV50_IR_OPCODE_CASE(LODQ, TXLQ);
800
801 NV50_IR_OPCODE_CASE(EMIT, EMIT);
802 NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
803
804 NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
805
806 NV50_IR_OPCODE_CASE(F2I, CVT);
807 NV50_IR_OPCODE_CASE(FSEQ, SET);
808 NV50_IR_OPCODE_CASE(FSGE, SET);
809 NV50_IR_OPCODE_CASE(FSLT, SET);
810 NV50_IR_OPCODE_CASE(FSNE, SET);
811 NV50_IR_OPCODE_CASE(IDIV, DIV);
812 NV50_IR_OPCODE_CASE(IMAX, MAX);
813 NV50_IR_OPCODE_CASE(IMIN, MIN);
814 NV50_IR_OPCODE_CASE(IABS, ABS);
815 NV50_IR_OPCODE_CASE(INEG, NEG);
816 NV50_IR_OPCODE_CASE(ISGE, SET);
817 NV50_IR_OPCODE_CASE(ISHR, SHR);
818 NV50_IR_OPCODE_CASE(ISLT, SET);
819 NV50_IR_OPCODE_CASE(F2U, CVT);
820 NV50_IR_OPCODE_CASE(U2F, CVT);
821 NV50_IR_OPCODE_CASE(UADD, ADD);
822 NV50_IR_OPCODE_CASE(UDIV, DIV);
823 NV50_IR_OPCODE_CASE(UMAD, MAD);
824 NV50_IR_OPCODE_CASE(UMAX, MAX);
825 NV50_IR_OPCODE_CASE(UMIN, MIN);
826 NV50_IR_OPCODE_CASE(UMOD, MOD);
827 NV50_IR_OPCODE_CASE(UMUL, MUL);
828 NV50_IR_OPCODE_CASE(USEQ, SET);
829 NV50_IR_OPCODE_CASE(USGE, SET);
830 NV50_IR_OPCODE_CASE(USHR, SHR);
831 NV50_IR_OPCODE_CASE(USLT, SET);
832 NV50_IR_OPCODE_CASE(USNE, SET);
833
834 NV50_IR_OPCODE_CASE(DABS, ABS);
835 NV50_IR_OPCODE_CASE(DNEG, NEG);
836 NV50_IR_OPCODE_CASE(DADD, ADD);
837 NV50_IR_OPCODE_CASE(DMUL, MUL);
838 NV50_IR_OPCODE_CASE(DDIV, DIV);
839 NV50_IR_OPCODE_CASE(DMAX, MAX);
840 NV50_IR_OPCODE_CASE(DMIN, MIN);
841 NV50_IR_OPCODE_CASE(DSLT, SET);
842 NV50_IR_OPCODE_CASE(DSGE, SET);
843 NV50_IR_OPCODE_CASE(DSEQ, SET);
844 NV50_IR_OPCODE_CASE(DSNE, SET);
845 NV50_IR_OPCODE_CASE(DRCP, RCP);
846 NV50_IR_OPCODE_CASE(DSQRT, SQRT);
847 NV50_IR_OPCODE_CASE(DMAD, MAD);
848 NV50_IR_OPCODE_CASE(DFMA, FMA);
849 NV50_IR_OPCODE_CASE(D2I, CVT);
850 NV50_IR_OPCODE_CASE(D2U, CVT);
851 NV50_IR_OPCODE_CASE(I2D, CVT);
852 NV50_IR_OPCODE_CASE(U2D, CVT);
853 NV50_IR_OPCODE_CASE(DRSQ, RSQ);
854 NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
855 NV50_IR_OPCODE_CASE(DCEIL, CEIL);
856 NV50_IR_OPCODE_CASE(DFLR, FLOOR);
857 NV50_IR_OPCODE_CASE(DROUND, CVT);
858
859 NV50_IR_OPCODE_CASE(U64SEQ, SET);
860 NV50_IR_OPCODE_CASE(U64SNE, SET);
861 NV50_IR_OPCODE_CASE(U64SLT, SET);
862 NV50_IR_OPCODE_CASE(U64SGE, SET);
863 NV50_IR_OPCODE_CASE(I64SLT, SET);
864 NV50_IR_OPCODE_CASE(I64SGE, SET);
865 NV50_IR_OPCODE_CASE(I2I64, CVT);
866 NV50_IR_OPCODE_CASE(U2I64, CVT);
867 NV50_IR_OPCODE_CASE(F2I64, CVT);
868 NV50_IR_OPCODE_CASE(F2U64, CVT);
869 NV50_IR_OPCODE_CASE(D2I64, CVT);
870 NV50_IR_OPCODE_CASE(D2U64, CVT);
871 NV50_IR_OPCODE_CASE(I642F, CVT);
872 NV50_IR_OPCODE_CASE(U642F, CVT);
873 NV50_IR_OPCODE_CASE(I642D, CVT);
874 NV50_IR_OPCODE_CASE(U642D, CVT);
875
876 NV50_IR_OPCODE_CASE(I64MIN, MIN);
877 NV50_IR_OPCODE_CASE(U64MIN, MIN);
878 NV50_IR_OPCODE_CASE(I64MAX, MAX);
879 NV50_IR_OPCODE_CASE(U64MAX, MAX);
880 NV50_IR_OPCODE_CASE(I64ABS, ABS);
881 NV50_IR_OPCODE_CASE(I64NEG, NEG);
882 NV50_IR_OPCODE_CASE(U64ADD, ADD);
883 NV50_IR_OPCODE_CASE(U64MUL, MUL);
884 NV50_IR_OPCODE_CASE(U64SHL, SHL);
885 NV50_IR_OPCODE_CASE(I64SHR, SHR);
886 NV50_IR_OPCODE_CASE(U64SHR, SHR);
887
888 NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
889 NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
890
891 NV50_IR_OPCODE_CASE(SAMPLE, TEX);
892 NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
893 NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
894 NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
895 NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
896 NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
897 NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
898 NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
899 NV50_IR_OPCODE_CASE(GATHER4, TXG);
900 NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
901
902 NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
903 NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
904 NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
905 NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
906 NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
907 NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
908 NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
909 NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
910 NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
911 NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
912 NV50_IR_OPCODE_CASE(ATOMFADD, ATOM);
913 NV50_IR_OPCODE_CASE(ATOMDEC_WRAP, ATOM);
914 NV50_IR_OPCODE_CASE(ATOMINC_WRAP, ATOM);
915
916 NV50_IR_OPCODE_CASE(TEX2, TEX);
917 NV50_IR_OPCODE_CASE(TXB2, TXB);
918 NV50_IR_OPCODE_CASE(TXL2, TXL);
919
920 NV50_IR_OPCODE_CASE(IBFE, EXTBF);
921 NV50_IR_OPCODE_CASE(UBFE, EXTBF);
922 NV50_IR_OPCODE_CASE(BFI, INSBF);
923 NV50_IR_OPCODE_CASE(BREV, EXTBF);
924 NV50_IR_OPCODE_CASE(POPC, POPCNT);
925 NV50_IR_OPCODE_CASE(LSB, BFIND);
926 NV50_IR_OPCODE_CASE(IMSB, BFIND);
927 NV50_IR_OPCODE_CASE(UMSB, BFIND);
928
929 NV50_IR_OPCODE_CASE(VOTE_ALL, VOTE);
930 NV50_IR_OPCODE_CASE(VOTE_ANY, VOTE);
931 NV50_IR_OPCODE_CASE(VOTE_EQ, VOTE);
932
933 NV50_IR_OPCODE_CASE(BALLOT, VOTE);
934 NV50_IR_OPCODE_CASE(READ_INVOC, SHFL);
935 NV50_IR_OPCODE_CASE(READ_FIRST, SHFL);
936
937 NV50_IR_OPCODE_CASE(END, EXIT);
938
939 default:
940 return nv50_ir::OP_NOP;
941 }
942 }
943
opcodeToSubOp(uint opcode)944 static uint16_t opcodeToSubOp(uint opcode)
945 {
946 switch (opcode) {
947 case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
948 case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
949 case TGSI_OPCODE_ATOMCAS: return NV50_IR_SUBOP_ATOM_CAS;
950 case TGSI_OPCODE_ATOMAND: return NV50_IR_SUBOP_ATOM_AND;
951 case TGSI_OPCODE_ATOMOR: return NV50_IR_SUBOP_ATOM_OR;
952 case TGSI_OPCODE_ATOMXOR: return NV50_IR_SUBOP_ATOM_XOR;
953 case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
954 case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
955 case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
956 case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
957 case TGSI_OPCODE_ATOMFADD: return NV50_IR_SUBOP_ATOM_ADD;
958 case TGSI_OPCODE_ATOMDEC_WRAP: return NV50_IR_SUBOP_ATOM_DEC;
959 case TGSI_OPCODE_ATOMINC_WRAP: return NV50_IR_SUBOP_ATOM_INC;
960 case TGSI_OPCODE_IMUL_HI:
961 case TGSI_OPCODE_UMUL_HI:
962 return NV50_IR_SUBOP_MUL_HIGH;
963 case TGSI_OPCODE_VOTE_ALL: return NV50_IR_SUBOP_VOTE_ALL;
964 case TGSI_OPCODE_VOTE_ANY: return NV50_IR_SUBOP_VOTE_ANY;
965 case TGSI_OPCODE_VOTE_EQ: return NV50_IR_SUBOP_VOTE_UNI;
966 default:
967 return 0;
968 }
969 }
970
checkDstSrcAliasing() const971 bool Instruction::checkDstSrcAliasing() const
972 {
973 if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
974 return false;
975
976 for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
977 if (insn->Src[s].Register.File == TGSI_FILE_NULL)
978 break;
979 if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
980 insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
981 return true;
982 }
983 return false;
984 }
985
986 class Source
987 {
988 public:
989 Source(struct nv50_ir_prog_info *, struct nv50_ir_prog_info_out *, nv50_ir::Program *);
990 ~Source();
991
992 public:
993 bool scanSource();
fileSize(unsigned file) const994 unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
995
996 public:
997 struct tgsi_shader_info scan;
998 struct tgsi_full_instruction *insns;
999 const struct tgsi_token *tokens;
1000 struct nv50_ir_prog_info *info;
1001 struct nv50_ir_prog_info_out *info_out;
1002
1003 nv50_ir::DynArray tempArrays;
1004 nv50_ir::DynArray immdArrays;
1005
1006 typedef nv50_ir::BuildUtil::Location Location;
1007 // these registers are per-subroutine, cannot be used for parameter passing
1008 std::set<Location> locals;
1009
1010 std::set<int> indirectTempArrays;
1011 std::map<int, int> indirectTempOffsets;
1012 std::map<int, std::pair<int, int> > tempArrayInfo;
1013 std::vector<int> tempArrayId;
1014
1015 int clipVertexOutput;
1016
1017 struct TextureView {
1018 uint8_t target; // TGSI_TEXTURE_*
1019 };
1020 std::vector<TextureView> textureViews;
1021
1022 /*
1023 struct Resource {
1024 uint8_t target; // TGSI_TEXTURE_*
1025 bool raw;
1026 uint8_t slot; // $surface index
1027 };
1028 std::vector<Resource> resources;
1029 */
1030
1031 struct MemoryFile {
1032 uint8_t mem_type; // TGSI_MEMORY_TYPE_*
1033 };
1034 std::vector<MemoryFile> memoryFiles;
1035
1036 std::vector<bool> bufferAtomics;
1037
1038 struct {
1039 uint16_t count; /* count of inline immediates */
1040 uint32_t *data; /* inline immediate data */
1041 } immd;
1042
1043 private:
1044 nv50_ir::Program *prog;
1045 int inferSysValDirection(unsigned sn) const;
1046 bool scanDeclaration(const struct tgsi_full_declaration *);
1047 bool scanInstruction(const struct tgsi_full_instruction *);
1048 void scanInstructionSrc(const Instruction& insn,
1049 const Instruction::SrcRegister& src,
1050 unsigned mask);
1051 void scanProperty(const struct tgsi_full_property *);
1052 void scanImmediate(const struct tgsi_full_immediate *);
1053
1054 inline bool isEdgeFlagPassthrough(const Instruction&) const;
1055 };
1056
Source(struct nv50_ir_prog_info * info,struct nv50_ir_prog_info_out * info_out,nv50_ir::Program * prog)1057 Source::Source(struct nv50_ir_prog_info *info, struct nv50_ir_prog_info_out *info_out,
1058 nv50_ir::Program *prog)
1059 : insns(NULL), info(info), info_out(info_out), clipVertexOutput(-1), prog(prog)
1060 {
1061 tokens = (const struct tgsi_token *)info->bin.source;
1062
1063 if (info->dbgFlags & NV50_IR_DEBUG_BASIC)
1064 tgsi_dump(tokens, 0);
1065
1066 tgsi_scan_shader(tokens, &scan);
1067
1068 immd.count = 0;
1069 immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
1070 }
1071
~Source()1072 Source::~Source()
1073 {
1074 if (insns)
1075 FREE(insns);
1076
1077 if (immd.data)
1078 FREE(immd.data);
1079 }
1080
scanSource()1081 bool Source::scanSource()
1082 {
1083 unsigned insnCount = 0;
1084 struct tgsi_parse_context parse;
1085
1086 insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
1087 sizeof(insns[0]));
1088 if (!insns)
1089 return false;
1090
1091 textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
1092 //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
1093 tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
1094 memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
1095 bufferAtomics.resize(scan.file_max[TGSI_FILE_BUFFER] + 1);
1096
1097 info_out->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
1098 info_out->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
1099 info_out->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
1100
1101 if (info->type == PIPE_SHADER_FRAGMENT) {
1102 info_out->prop.fp.writesDepth = scan.writes_z;
1103 info_out->prop.fp.usesDiscard = scan.uses_kill || info->io.alphaRefBase;
1104 } else
1105 if (info->type == PIPE_SHADER_GEOMETRY) {
1106 info_out->prop.gp.instanceCount = 1; // default value
1107 }
1108
1109 info->io.viewportId = -1;
1110
1111 tgsi_parse_init(&parse, tokens);
1112 while (!tgsi_parse_end_of_tokens(&parse)) {
1113 tgsi_parse_token(&parse);
1114
1115 switch (parse.FullToken.Token.Type) {
1116 case TGSI_TOKEN_TYPE_IMMEDIATE:
1117 scanImmediate(&parse.FullToken.FullImmediate);
1118 break;
1119 case TGSI_TOKEN_TYPE_DECLARATION:
1120 scanDeclaration(&parse.FullToken.FullDeclaration);
1121 break;
1122 case TGSI_TOKEN_TYPE_INSTRUCTION:
1123 insns[insnCount++] = parse.FullToken.FullInstruction;
1124 scanInstruction(&parse.FullToken.FullInstruction);
1125 break;
1126 case TGSI_TOKEN_TYPE_PROPERTY:
1127 scanProperty(&parse.FullToken.FullProperty);
1128 break;
1129 default:
1130 INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
1131 break;
1132 }
1133 }
1134 tgsi_parse_free(&parse);
1135
1136 if (indirectTempArrays.size()) {
1137 int tempBase = 0;
1138 for (std::set<int>::const_iterator it = indirectTempArrays.begin();
1139 it != indirectTempArrays.end(); ++it) {
1140 std::pair<int, int>& info = tempArrayInfo[*it];
1141 indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
1142 tempBase += info.second;
1143 }
1144 info_out->bin.tlsSpace += tempBase * 16;
1145 }
1146
1147 if (info_out->io.genUserClip > 0) {
1148 info_out->io.clipDistances = info_out->io.genUserClip;
1149
1150 const unsigned int nOut = (info_out->io.genUserClip + 3) / 4;
1151
1152 for (unsigned int n = 0; n < nOut; ++n) {
1153 unsigned int i = info_out->numOutputs++;
1154 info_out->out[i].id = i;
1155 info_out->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
1156 info_out->out[i].si = n;
1157 info_out->out[i].mask = ((1 << info_out->io.clipDistances) - 1) >> (n * 4);
1158 }
1159 }
1160
1161 return info->assignSlots(info_out) == 0;
1162 }
1163
scanProperty(const struct tgsi_full_property * prop)1164 void Source::scanProperty(const struct tgsi_full_property *prop)
1165 {
1166 switch (prop->Property.PropertyName) {
1167 case TGSI_PROPERTY_GS_OUTPUT_PRIM:
1168 info_out->prop.gp.outputPrim = prop->u[0].Data;
1169 break;
1170 case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
1171 info_out->prop.gp.maxVertices = prop->u[0].Data;
1172 break;
1173 case TGSI_PROPERTY_GS_INVOCATIONS:
1174 info_out->prop.gp.instanceCount = prop->u[0].Data;
1175 break;
1176 case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1177 info_out->prop.fp.separateFragData = true;
1178 break;
1179 case TGSI_PROPERTY_FS_COORD_ORIGIN:
1180 case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1181 case TGSI_PROPERTY_FS_DEPTH_LAYOUT:
1182 case TGSI_PROPERTY_GS_INPUT_PRIM:
1183 // we don't care
1184 break;
1185 case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1186 info_out->io.genUserClip = -1;
1187 break;
1188 case TGSI_PROPERTY_TCS_VERTICES_OUT:
1189 info_out->prop.tp.outputPatchSize = prop->u[0].Data;
1190 break;
1191 case TGSI_PROPERTY_TES_PRIM_MODE:
1192 info_out->prop.tp.domain = prop->u[0].Data;
1193 break;
1194 case TGSI_PROPERTY_TES_SPACING:
1195 info_out->prop.tp.partitioning = prop->u[0].Data;
1196 break;
1197 case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1198 info_out->prop.tp.winding = prop->u[0].Data;
1199 break;
1200 case TGSI_PROPERTY_TES_POINT_MODE:
1201 if (prop->u[0].Data)
1202 info_out->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1203 else
1204 info_out->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1205 break;
1206 case TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH:
1207 info->prop.cp.numThreads[0] = prop->u[0].Data;
1208 break;
1209 case TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT:
1210 info->prop.cp.numThreads[1] = prop->u[0].Data;
1211 break;
1212 case TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH:
1213 info->prop.cp.numThreads[2] = prop->u[0].Data;
1214 break;
1215 case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1216 info_out->io.clipDistances = prop->u[0].Data;
1217 break;
1218 case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1219 info_out->io.cullDistances = prop->u[0].Data;
1220 break;
1221 case TGSI_PROPERTY_NEXT_SHADER:
1222 /* Do not need to know the next shader stage. */
1223 break;
1224 case TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL:
1225 info_out->prop.fp.earlyFragTests = prop->u[0].Data;
1226 break;
1227 case TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE:
1228 info_out->prop.fp.postDepthCoverage = prop->u[0].Data;
1229 break;
1230 case TGSI_PROPERTY_MUL_ZERO_WINS:
1231 info->io.mul_zero_wins = prop->u[0].Data;
1232 break;
1233 case TGSI_PROPERTY_LAYER_VIEWPORT_RELATIVE:
1234 info_out->io.layer_viewport_relative = prop->u[0].Data;
1235 break;
1236 default:
1237 INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1238 break;
1239 }
1240 }
1241
scanImmediate(const struct tgsi_full_immediate * imm)1242 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1243 {
1244 const unsigned n = immd.count++;
1245
1246 assert(n < scan.immediate_count);
1247
1248 for (int c = 0; c < 4; ++c)
1249 immd.data[n * 4 + c] = imm->u[c].Uint;
1250 }
1251
inferSysValDirection(unsigned sn) const1252 int Source::inferSysValDirection(unsigned sn) const
1253 {
1254 switch (sn) {
1255 case TGSI_SEMANTIC_INSTANCEID:
1256 case TGSI_SEMANTIC_VERTEXID:
1257 return 1;
1258 case TGSI_SEMANTIC_LAYER:
1259 #if 0
1260 case TGSI_SEMANTIC_VIEWPORTINDEX:
1261 return 0;
1262 #endif
1263 case TGSI_SEMANTIC_PRIMID:
1264 return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1265 default:
1266 return 0;
1267 }
1268 }
1269
scanDeclaration(const struct tgsi_full_declaration * decl)1270 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1271 {
1272 unsigned i, c;
1273 unsigned sn = TGSI_SEMANTIC_GENERIC;
1274 unsigned si = 0;
1275 const unsigned first = decl->Range.First, last = decl->Range.Last;
1276 const int arrayId = decl->Array.ArrayID;
1277
1278 if (decl->Declaration.Semantic) {
1279 sn = decl->Semantic.Name;
1280 si = decl->Semantic.Index;
1281 }
1282
1283 if (decl->Declaration.Local || decl->Declaration.File == TGSI_FILE_ADDRESS) {
1284 for (i = first; i <= last; ++i) {
1285 for (c = 0; c < 4; ++c) {
1286 locals.insert(
1287 Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1288 }
1289 }
1290 }
1291
1292 switch (decl->Declaration.File) {
1293 case TGSI_FILE_INPUT:
1294 if (info->type == PIPE_SHADER_VERTEX) {
1295 // all vertex attributes are equal
1296 for (i = first; i <= last; ++i) {
1297 info_out->in[i].sn = TGSI_SEMANTIC_GENERIC;
1298 info_out->in[i].si = i;
1299 }
1300 } else {
1301 for (i = first; i <= last; ++i, ++si) {
1302 info_out->in[i].id = i;
1303 info_out->in[i].sn = sn;
1304 info_out->in[i].si = si;
1305 if (info->type == PIPE_SHADER_FRAGMENT) {
1306 // translate interpolation mode
1307 switch (decl->Interp.Interpolate) {
1308 case TGSI_INTERPOLATE_CONSTANT:
1309 info_out->in[i].flat = 1;
1310 break;
1311 case TGSI_INTERPOLATE_COLOR:
1312 info_out->in[i].sc = 1;
1313 break;
1314 case TGSI_INTERPOLATE_LINEAR:
1315 info_out->in[i].linear = 1;
1316 break;
1317 default:
1318 break;
1319 }
1320 if (decl->Interp.Location)
1321 info_out->in[i].centroid = 1;
1322 }
1323
1324 if (sn == TGSI_SEMANTIC_PATCH)
1325 info_out->in[i].patch = 1;
1326 if (sn == TGSI_SEMANTIC_PATCH)
1327 info_out->numPatchConstants = MAX2(info_out->numPatchConstants, si + 1);
1328 }
1329 }
1330 break;
1331 case TGSI_FILE_OUTPUT:
1332 for (i = first; i <= last; ++i, ++si) {
1333 switch (sn) {
1334 case TGSI_SEMANTIC_POSITION:
1335 if (info->type == PIPE_SHADER_FRAGMENT)
1336 info_out->io.fragDepth = i;
1337 else
1338 if (clipVertexOutput < 0)
1339 clipVertexOutput = i;
1340 break;
1341 case TGSI_SEMANTIC_COLOR:
1342 if (info->type == PIPE_SHADER_FRAGMENT)
1343 info_out->prop.fp.numColourResults++;
1344 break;
1345 case TGSI_SEMANTIC_EDGEFLAG:
1346 info_out->io.edgeFlagOut = i;
1347 break;
1348 case TGSI_SEMANTIC_CLIPVERTEX:
1349 clipVertexOutput = i;
1350 break;
1351 case TGSI_SEMANTIC_CLIPDIST:
1352 info_out->io.genUserClip = -1;
1353 break;
1354 case TGSI_SEMANTIC_SAMPLEMASK:
1355 info_out->io.sampleMask = i;
1356 break;
1357 case TGSI_SEMANTIC_VIEWPORT_INDEX:
1358 info->io.viewportId = i;
1359 break;
1360 case TGSI_SEMANTIC_PATCH:
1361 info_out->numPatchConstants = MAX2(info_out->numPatchConstants, si + 1);
1362 /* fallthrough */
1363 case TGSI_SEMANTIC_TESSOUTER:
1364 case TGSI_SEMANTIC_TESSINNER:
1365 info_out->out[i].patch = 1;
1366 break;
1367 default:
1368 break;
1369 }
1370 info_out->out[i].id = i;
1371 info_out->out[i].sn = sn;
1372 info_out->out[i].si = si;
1373 }
1374 break;
1375 case TGSI_FILE_SYSTEM_VALUE:
1376 switch (sn) {
1377 case TGSI_SEMANTIC_INSTANCEID:
1378 info_out->io.instanceId = first;
1379 break;
1380 case TGSI_SEMANTIC_VERTEXID:
1381 info_out->io.vertexId = first;
1382 break;
1383 case TGSI_SEMANTIC_BASEVERTEX:
1384 case TGSI_SEMANTIC_BASEINSTANCE:
1385 case TGSI_SEMANTIC_DRAWID:
1386 info_out->prop.vp.usesDrawParameters = true;
1387 break;
1388 case TGSI_SEMANTIC_SAMPLEID:
1389 case TGSI_SEMANTIC_SAMPLEPOS:
1390 prog->persampleInvocation = true;
1391 break;
1392 case TGSI_SEMANTIC_SAMPLEMASK:
1393 info_out->prop.fp.usesSampleMaskIn = true;
1394 break;
1395 default:
1396 break;
1397 }
1398 for (i = first; i <= last; ++i, ++si) {
1399 info_out->sv[i].sn = sn;
1400 info_out->sv[i].si = si;
1401 info_out->sv[i].input = inferSysValDirection(sn);
1402
1403 switch (sn) {
1404 case TGSI_SEMANTIC_TESSOUTER:
1405 case TGSI_SEMANTIC_TESSINNER:
1406 info_out->sv[i].patch = 1;
1407 break;
1408 }
1409 }
1410 break;
1411 /*
1412 case TGSI_FILE_RESOURCE:
1413 for (i = first; i <= last; ++i) {
1414 resources[i].target = decl->Resource.Resource;
1415 resources[i].raw = decl->Resource.Raw;
1416 resources[i].slot = i;
1417 }
1418 break;
1419 */
1420 case TGSI_FILE_SAMPLER_VIEW:
1421 for (i = first; i <= last; ++i)
1422 textureViews[i].target = decl->SamplerView.Resource;
1423 break;
1424 case TGSI_FILE_MEMORY:
1425 for (i = first; i <= last; ++i)
1426 memoryFiles[i].mem_type = decl->Declaration.MemType;
1427 break;
1428 case TGSI_FILE_NULL:
1429 case TGSI_FILE_TEMPORARY:
1430 for (i = first; i <= last; ++i)
1431 tempArrayId[i] = arrayId;
1432 if (arrayId)
1433 tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1434 first, last - first + 1)));
1435 break;
1436 case TGSI_FILE_BUFFER:
1437 for (i = first; i <= last; ++i)
1438 bufferAtomics[i] = decl->Declaration.Atomic;
1439 break;
1440 case TGSI_FILE_ADDRESS:
1441 case TGSI_FILE_CONSTANT:
1442 case TGSI_FILE_IMMEDIATE:
1443 case TGSI_FILE_SAMPLER:
1444 case TGSI_FILE_IMAGE:
1445 break;
1446 default:
1447 ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1448 return false;
1449 }
1450 return true;
1451 }
1452
isEdgeFlagPassthrough(const Instruction & insn) const1453 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1454 {
1455 return insn.getOpcode() == TGSI_OPCODE_MOV &&
1456 insn.getDst(0).getIndex(0) == info_out->io.edgeFlagOut &&
1457 insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1458 }
1459
scanInstructionSrc(const Instruction & insn,const Instruction::SrcRegister & src,unsigned mask)1460 void Source::scanInstructionSrc(const Instruction& insn,
1461 const Instruction::SrcRegister& src,
1462 unsigned mask)
1463 {
1464 if (src.getFile() == TGSI_FILE_TEMPORARY) {
1465 if (src.isIndirect(0))
1466 indirectTempArrays.insert(src.getArrayId());
1467 } else
1468 if (src.getFile() == TGSI_FILE_OUTPUT) {
1469 if (src.isIndirect(0)) {
1470 // We don't know which one is accessed, just mark everything for
1471 // reading. This is an extremely unlikely occurrence.
1472 for (unsigned i = 0; i < info_out->numOutputs; ++i)
1473 info_out->out[i].oread = 1;
1474 } else {
1475 info_out->out[src.getIndex(0)].oread = 1;
1476 }
1477 }
1478 if (src.getFile() == TGSI_FILE_SYSTEM_VALUE) {
1479 if (info_out->sv[src.getIndex(0)].sn == TGSI_SEMANTIC_SAMPLEPOS)
1480 info_out->prop.fp.readsSampleLocations = true;
1481 }
1482 if (src.getFile() != TGSI_FILE_INPUT)
1483 return;
1484
1485 if (src.isIndirect(0)) {
1486 for (unsigned i = 0; i < info_out->numInputs; ++i)
1487 info_out->in[i].mask = 0xf;
1488 } else {
1489 const int i = src.getIndex(0);
1490 for (unsigned c = 0; c < 4; ++c) {
1491 if (!(mask & (1 << c)))
1492 continue;
1493 int k = src.getSwizzle(c);
1494 if (k <= TGSI_SWIZZLE_W)
1495 info_out->in[i].mask |= 1 << k;
1496 }
1497 switch (info_out->in[i].sn) {
1498 case TGSI_SEMANTIC_PSIZE:
1499 case TGSI_SEMANTIC_PRIMID:
1500 case TGSI_SEMANTIC_FOG:
1501 info_out->in[i].mask &= 0x1;
1502 break;
1503 case TGSI_SEMANTIC_PCOORD:
1504 info_out->in[i].mask &= 0x3;
1505 break;
1506 default:
1507 break;
1508 }
1509 }
1510 }
1511
scanInstruction(const struct tgsi_full_instruction * inst)1512 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1513 {
1514 Instruction insn(inst);
1515
1516 if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1517 info_out->numBarriers = 1;
1518
1519 if (insn.getOpcode() == TGSI_OPCODE_FBFETCH)
1520 info_out->prop.fp.readsFramebuffer = true;
1521
1522 if (insn.getOpcode() == TGSI_OPCODE_INTERP_SAMPLE)
1523 info_out->prop.fp.readsSampleLocations = true;
1524
1525 if (insn.getOpcode() == TGSI_OPCODE_DEMOTE)
1526 info_out->prop.fp.usesDiscard = true;
1527
1528 if (insn.dstCount()) {
1529 Instruction::DstRegister dst = insn.getDst(0);
1530
1531 if (insn.getOpcode() == TGSI_OPCODE_STORE &&
1532 dst.getFile() != TGSI_FILE_MEMORY) {
1533 info_out->io.globalAccess |= 0x2;
1534
1535 if (dst.getFile() == TGSI_FILE_INPUT) {
1536 // TODO: Handle indirect somehow?
1537 const int i = dst.getIndex(0);
1538 info_out->in[i].mask |= 1;
1539 }
1540 }
1541
1542 if (dst.getFile() == TGSI_FILE_OUTPUT) {
1543 if (dst.isIndirect(0))
1544 for (unsigned i = 0; i < info_out->numOutputs; ++i)
1545 info_out->out[i].mask = 0xf;
1546 else
1547 info_out->out[dst.getIndex(0)].mask |= dst.getMask();
1548
1549 if (info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1550 info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1551 info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1552 info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1553 info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1554 info_out->out[dst.getIndex(0)].mask &= 1;
1555
1556 if (isEdgeFlagPassthrough(insn))
1557 info_out->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1558 } else
1559 if (dst.getFile() == TGSI_FILE_TEMPORARY) {
1560 if (dst.isIndirect(0))
1561 indirectTempArrays.insert(dst.getArrayId());
1562 } else
1563 if (dst.getFile() == TGSI_FILE_BUFFER ||
1564 dst.getFile() == TGSI_FILE_IMAGE ||
1565 (dst.getFile() == TGSI_FILE_MEMORY &&
1566 memoryFiles[dst.getIndex(0)].mem_type == TGSI_MEMORY_TYPE_GLOBAL)) {
1567 info_out->io.globalAccess |= 0x2;
1568 }
1569 }
1570
1571 if (insn.srcCount() && (
1572 insn.getSrc(0).getFile() != TGSI_FILE_MEMORY ||
1573 memoryFiles[insn.getSrc(0).getIndex(0)].mem_type ==
1574 TGSI_MEMORY_TYPE_GLOBAL)) {
1575 switch (insn.getOpcode()) {
1576 case TGSI_OPCODE_ATOMUADD:
1577 case TGSI_OPCODE_ATOMXCHG:
1578 case TGSI_OPCODE_ATOMCAS:
1579 case TGSI_OPCODE_ATOMAND:
1580 case TGSI_OPCODE_ATOMOR:
1581 case TGSI_OPCODE_ATOMXOR:
1582 case TGSI_OPCODE_ATOMUMIN:
1583 case TGSI_OPCODE_ATOMIMIN:
1584 case TGSI_OPCODE_ATOMUMAX:
1585 case TGSI_OPCODE_ATOMIMAX:
1586 case TGSI_OPCODE_ATOMFADD:
1587 case TGSI_OPCODE_ATOMDEC_WRAP:
1588 case TGSI_OPCODE_ATOMINC_WRAP:
1589 case TGSI_OPCODE_LOAD:
1590 info_out->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1591 0x1 : 0x2;
1592 break;
1593 }
1594 }
1595
1596
1597 for (unsigned s = 0; s < insn.srcCount(); ++s)
1598 scanInstructionSrc(insn, insn.getSrc(s), insn.srcMask(s));
1599
1600 for (unsigned s = 0; s < insn.getNumTexOffsets(); ++s)
1601 scanInstructionSrc(insn, insn.getTexOffset(s), insn.texOffsetMask());
1602
1603 return true;
1604 }
1605
1606 nv50_ir::TexInstruction::Target
getTexture(const tgsi::Source * code,int s) const1607 Instruction::getTexture(const tgsi::Source *code, int s) const
1608 {
1609 // XXX: indirect access
1610 unsigned int r;
1611
1612 switch (getSrc(s).getFile()) {
1613 /*
1614 case TGSI_FILE_RESOURCE:
1615 r = getSrc(s).getIndex(0);
1616 return translateTexture(code->resources.at(r).target);
1617 */
1618 case TGSI_FILE_SAMPLER_VIEW:
1619 r = getSrc(s).getIndex(0);
1620 return translateTexture(code->textureViews.at(r).target);
1621 default:
1622 return translateTexture(insn->Texture.Texture);
1623 }
1624 }
1625
1626 } // namespace tgsi
1627
1628 namespace {
1629
1630 using namespace nv50_ir;
1631
1632 class Converter : public ConverterCommon
1633 {
1634 public:
1635 Converter(Program *, const tgsi::Source *, nv50_ir_prog_info_out *);
1636 ~Converter();
1637
1638 bool run();
1639
1640 private:
1641 Value *shiftAddress(Value *);
1642 Value *getVertexBase(int s);
1643 Value *getOutputBase(int s);
1644 DataArray *getArrayForFile(unsigned file, int idx);
1645 Value *fetchSrc(int s, int c);
1646 Value *fetchDst(int d, int c);
1647 Value *acquireDst(int d, int c);
1648 void storeDst(int d, int c, Value *);
1649
1650 Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1651 void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1652 Value *val, Value *ptr);
1653
1654 void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1655 Value *applySrcMod(Value *, int s, int c);
1656
1657 Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1658 Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1659 Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1660
1661 bool isSubGroupMask(uint8_t semantic);
1662
1663 bool handleInstruction(const struct tgsi_full_instruction *);
1664 void exportOutputs();
1665 inline bool isEndOfSubroutine(uint ip);
1666
1667 void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1668
1669 // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1670 void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1671 void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1672 void handleTXF(Value *dst0[4], int R, int L_M);
1673 void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1674 void handleFBFETCH(Value *dst0[4]);
1675 void handleLIT(Value *dst0[4]);
1676
1677 // Symbol *getResourceBase(int r);
1678 void getImageCoords(std::vector<Value *>&, int s);
1679
1680 void handleLOAD(Value *dst0[4]);
1681 void handleSTORE();
1682 void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1683
1684 void handleINTERP(Value *dst0[4]);
1685
1686 Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1687
1688 void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1689
1690 Value *buildDot(int dim);
1691
1692 class BindArgumentsPass : public Pass {
1693 public:
BindArgumentsPass(Converter & conv)1694 BindArgumentsPass(Converter &conv) : conv(conv) { }
1695
1696 private:
1697 Converter &conv;
1698 Subroutine *sub;
1699
1700 inline const Location *getValueLocation(Subroutine *, Value *);
1701
1702 template<typename T> inline void
1703 updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1704 T (Function::*proto));
1705
1706 template<typename T> inline void
1707 updatePrototype(BitSet *set, void (Function::*updateSet)(),
1708 T (Function::*proto));
1709
1710 protected:
1711 bool visit(Function *);
visit(BasicBlock * bb)1712 bool visit(BasicBlock *bb) { return false; }
1713 };
1714
1715 private:
1716 const tgsi::Source *code;
1717
1718 uint ip; // instruction pointer
1719
1720 tgsi::Instruction tgsi;
1721
1722 DataType dstTy;
1723 DataType srcTy;
1724
1725 DataArray tData; // TGSI_FILE_TEMPORARY
1726 DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1727 DataArray aData; // TGSI_FILE_ADDRESS
1728 DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1729
1730 Value *zero;
1731
1732 Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1733 uint8_t vtxBaseValid;
1734
1735 Stack condBBs; // fork BB, then else clause BB
1736 Stack joinBBs; // fork BB, for inserting join ops on ENDIF
1737 Stack loopBBs; // loop headers
1738 Stack breakBBs; // end of / after loop
1739
1740 Value *viewport;
1741 };
1742
1743 Symbol *
srcToSym(tgsi::Instruction::SrcRegister src,int c)1744 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1745 {
1746 const int swz = src.getSwizzle(c);
1747
1748 /* TODO: Use Array ID when it's available for the index */
1749 return makeSym(src.getFile(),
1750 src.is2D() ? src.getIndex(1) : 0,
1751 src.getIndex(0), swz,
1752 src.getIndex(0) * 16 + swz * 4);
1753 }
1754
1755 Symbol *
dstToSym(tgsi::Instruction::DstRegister dst,int c)1756 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1757 {
1758 /* TODO: Use Array ID when it's available for the index */
1759 return makeSym(dst.getFile(),
1760 dst.is2D() ? dst.getIndex(1) : 0,
1761 dst.getIndex(0), c,
1762 dst.getIndex(0) * 16 + c * 4);
1763 }
1764
1765 Symbol *
makeSym(uint tgsiFile,int fileIdx,int idx,int c,uint32_t address)1766 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1767 {
1768 Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1769
1770 sym->reg.fileIndex = fileIdx;
1771
1772 if (tgsiFile == TGSI_FILE_MEMORY) {
1773 switch (code->memoryFiles[fileIdx].mem_type) {
1774 case TGSI_MEMORY_TYPE_GLOBAL:
1775 /* No-op this is the default for TGSI_FILE_MEMORY */
1776 sym->setFile(FILE_MEMORY_GLOBAL);
1777 break;
1778 case TGSI_MEMORY_TYPE_SHARED:
1779 sym->setFile(FILE_MEMORY_SHARED);
1780 break;
1781 case TGSI_MEMORY_TYPE_INPUT:
1782 assert(prog->getType() == Program::TYPE_COMPUTE);
1783 assert(idx == -1);
1784 sym->setFile(FILE_SHADER_INPUT);
1785 address += info->prop.cp.inputOffset;
1786 break;
1787 default:
1788 assert(0); /* TODO: Add support for global and private memory */
1789 }
1790 }
1791
1792 if (idx >= 0) {
1793 if (sym->reg.file == FILE_SHADER_INPUT)
1794 sym->setOffset(info_out->in[idx].slot[c] * 4);
1795 else
1796 if (sym->reg.file == FILE_SHADER_OUTPUT)
1797 sym->setOffset(info_out->out[idx].slot[c] * 4);
1798 else
1799 if (sym->reg.file == FILE_SYSTEM_VALUE)
1800 sym->setSV(tgsi::translateSysVal(info_out->sv[idx].sn), c);
1801 else
1802 sym->setOffset(address);
1803 } else {
1804 sym->setOffset(address);
1805 }
1806 return sym;
1807 }
1808
1809 Value *
interpolate(tgsi::Instruction::SrcRegister src,int c,Value * ptr)1810 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1811 {
1812 operation op;
1813
1814 // XXX: no way to know interpolation mode if we don't know what's accessed
1815 const uint8_t mode = translateInterpMode(&info_out->in[ptr ? 0 :
1816 src.getIndex(0)], op);
1817
1818 Instruction *insn = new_Instruction(func, op, TYPE_F32);
1819
1820 insn->setDef(0, getScratch());
1821 insn->setSrc(0, srcToSym(src, c));
1822 if (op == OP_PINTERP)
1823 insn->setSrc(1, fragCoord[3]);
1824 if (ptr)
1825 insn->setIndirect(0, 0, ptr);
1826
1827 insn->setInterpolate(mode);
1828
1829 bb->insertTail(insn);
1830 return insn->getDef(0);
1831 }
1832
1833 Value *
applySrcMod(Value * val,int s,int c)1834 Converter::applySrcMod(Value *val, int s, int c)
1835 {
1836 Modifier m = tgsi.getSrc(s).getMod(c);
1837 DataType ty = tgsi.inferSrcType();
1838
1839 if (m & Modifier(NV50_IR_MOD_ABS))
1840 val = mkOp1v(OP_ABS, ty, getScratch(), val);
1841
1842 if (m & Modifier(NV50_IR_MOD_NEG))
1843 val = mkOp1v(OP_NEG, ty, getScratch(), val);
1844
1845 return val;
1846 }
1847
1848 Value *
getVertexBase(int s)1849 Converter::getVertexBase(int s)
1850 {
1851 assert(s < 5);
1852 if (!(vtxBaseValid & (1 << s))) {
1853 const int index = tgsi.getSrc(s).getIndex(1);
1854 Value *rel = NULL;
1855 if (tgsi.getSrc(s).isIndirect(1))
1856 rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1857 vtxBaseValid |= 1 << s;
1858 vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1859 mkImm(index), rel);
1860 }
1861 return vtxBase[s];
1862 }
1863
1864 Value *
getOutputBase(int s)1865 Converter::getOutputBase(int s)
1866 {
1867 assert(s < 5);
1868 if (!(vtxBaseValid & (1 << s))) {
1869 Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1870 if (tgsi.getSrc(s).isIndirect(1))
1871 offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1872 fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1873 offset);
1874 vtxBaseValid |= 1 << s;
1875 vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1876 }
1877 return vtxBase[s];
1878 }
1879
1880 Value *
fetchSrc(int s,int c)1881 Converter::fetchSrc(int s, int c)
1882 {
1883 Value *res;
1884 Value *ptr = NULL, *dimRel = NULL;
1885
1886 tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1887
1888 if (src.isIndirect(0))
1889 ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1890
1891 if (src.is2D()) {
1892 switch (src.getFile()) {
1893 case TGSI_FILE_OUTPUT:
1894 dimRel = getOutputBase(s);
1895 break;
1896 case TGSI_FILE_INPUT:
1897 dimRel = getVertexBase(s);
1898 break;
1899 case TGSI_FILE_CONSTANT:
1900 // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1901 if (src.isIndirect(1))
1902 dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1903 break;
1904 default:
1905 break;
1906 }
1907 }
1908
1909 res = fetchSrc(src, c, ptr);
1910
1911 if (dimRel)
1912 res->getInsn()->setIndirect(0, 1, dimRel);
1913
1914 return applySrcMod(res, s, c);
1915 }
1916
1917 Value *
fetchDst(int d,int c)1918 Converter::fetchDst(int d, int c)
1919 {
1920 Value *res;
1921 Value *ptr = NULL, *dimRel = NULL;
1922
1923 tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1924
1925 if (dst.isIndirect(0))
1926 ptr = fetchSrc(dst.getIndirect(0), 0, NULL);
1927
1928 if (dst.is2D()) {
1929 switch (dst.getFile()) {
1930 case TGSI_FILE_OUTPUT:
1931 assert(0); // TODO
1932 dimRel = NULL;
1933 break;
1934 case TGSI_FILE_INPUT:
1935 assert(0); // TODO
1936 dimRel = NULL;
1937 break;
1938 case TGSI_FILE_CONSTANT:
1939 // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1940 if (dst.isIndirect(1))
1941 dimRel = fetchSrc(dst.getIndirect(1), 0, 0);
1942 break;
1943 default:
1944 break;
1945 }
1946 }
1947
1948 struct tgsi_full_src_register fsr = dst.asSrc();
1949 tgsi::Instruction::SrcRegister src(&fsr);
1950 res = fetchSrc(src, c, ptr);
1951
1952 if (dimRel)
1953 res->getInsn()->setIndirect(0, 1, dimRel);
1954
1955 return res;
1956 }
1957
1958 Converter::DataArray *
getArrayForFile(unsigned file,int idx)1959 Converter::getArrayForFile(unsigned file, int idx)
1960 {
1961 switch (file) {
1962 case TGSI_FILE_TEMPORARY:
1963 return idx == 0 ? &tData : &lData;
1964 case TGSI_FILE_ADDRESS:
1965 return &aData;
1966 case TGSI_FILE_OUTPUT:
1967 assert(prog->getType() == Program::TYPE_FRAGMENT);
1968 return &oData;
1969 default:
1970 assert(!"invalid/unhandled TGSI source file");
1971 return NULL;
1972 }
1973 }
1974
1975 Value *
shiftAddress(Value * index)1976 Converter::shiftAddress(Value *index)
1977 {
1978 if (!index)
1979 return NULL;
1980 return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1981 }
1982
1983 void
adjustTempIndex(int arrayId,int & idx,int & idx2d) const1984 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1985 {
1986 std::map<int, int>::const_iterator it =
1987 code->indirectTempOffsets.find(arrayId);
1988 if (it == code->indirectTempOffsets.end())
1989 return;
1990
1991 idx2d = 1;
1992 idx += it->second;
1993 }
1994
1995 bool
isSubGroupMask(uint8_t semantic)1996 Converter::isSubGroupMask(uint8_t semantic)
1997 {
1998 switch (semantic) {
1999 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2000 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2001 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2002 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2003 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2004 return true;
2005 default:
2006 return false;
2007 }
2008 }
2009
2010 Value *
fetchSrc(tgsi::Instruction::SrcRegister src,int c,Value * ptr)2011 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
2012 {
2013 int idx2d = src.is2D() ? src.getIndex(1) : 0;
2014 int idx = src.getIndex(0);
2015 const int swz = src.getSwizzle(c);
2016 Instruction *ld;
2017
2018 switch (src.getFile()) {
2019 case TGSI_FILE_IMMEDIATE:
2020 assert(!ptr);
2021 return loadImm(NULL, code->immd.data[idx * 4 + swz]);
2022 case TGSI_FILE_CONSTANT:
2023 return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
2024 case TGSI_FILE_INPUT:
2025 if (prog->getType() == Program::TYPE_FRAGMENT) {
2026 // don't load masked inputs, won't be assigned a slot
2027 if (!ptr && !(info_out->in[idx].mask & (1 << swz)))
2028 return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
2029 return interpolate(src, c, shiftAddress(ptr));
2030 } else
2031 if (prog->getType() == Program::TYPE_GEOMETRY) {
2032 if (!ptr && info_out->in[idx].sn == TGSI_SEMANTIC_PRIMID)
2033 return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
2034 // XXX: This is going to be a problem with scalar arrays, i.e. when
2035 // we cannot assume that the address is given in units of vec4.
2036 //
2037 // nv50 and nvc0 need different things here, so let the lowering
2038 // passes decide what to do with the address
2039 if (ptr)
2040 return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
2041 }
2042 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2043 ld->perPatch = info_out->in[idx].patch;
2044 return ld->getDef(0);
2045 case TGSI_FILE_OUTPUT:
2046 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2047 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2048 ld->perPatch = info_out->out[idx].patch;
2049 return ld->getDef(0);
2050 case TGSI_FILE_SYSTEM_VALUE:
2051 assert(!ptr);
2052 if (info_out->sv[idx].sn == TGSI_SEMANTIC_THREAD_ID &&
2053 info->prop.cp.numThreads[swz] == 1)
2054 return loadImm(NULL, 0u);
2055 if (isSubGroupMask(info_out->sv[idx].sn) && swz > 0)
2056 return loadImm(NULL, 0u);
2057 if (info_out->sv[idx].sn == TGSI_SEMANTIC_SUBGROUP_SIZE)
2058 return loadImm(NULL, 32u);
2059 ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
2060 ld->perPatch = info_out->sv[idx].patch;
2061 return ld->getDef(0);
2062 case TGSI_FILE_TEMPORARY: {
2063 int arrayid = src.getArrayId();
2064 if (!arrayid)
2065 arrayid = code->tempArrayId[idx];
2066 adjustTempIndex(arrayid, idx, idx2d);
2067 }
2068 /* fallthrough */
2069 default:
2070 return getArrayForFile(src.getFile(), idx2d)->load(
2071 sub.cur->values, idx, swz, shiftAddress(ptr));
2072 }
2073 }
2074
2075 Value *
acquireDst(int d,int c)2076 Converter::acquireDst(int d, int c)
2077 {
2078 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2079 const unsigned f = dst.getFile();
2080 int idx = dst.getIndex(0);
2081 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2082
2083 if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY ||
2084 f == TGSI_FILE_IMAGE)
2085 return NULL;
2086
2087 if (dst.isIndirect(0) ||
2088 f == TGSI_FILE_SYSTEM_VALUE ||
2089 (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
2090 return getScratch();
2091
2092 if (f == TGSI_FILE_TEMPORARY) {
2093 int arrayid = dst.getArrayId();
2094 if (!arrayid)
2095 arrayid = code->tempArrayId[idx];
2096 adjustTempIndex(arrayid, idx, idx2d);
2097 }
2098
2099 return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
2100 }
2101
2102 void
storeDst(int d,int c,Value * val)2103 Converter::storeDst(int d, int c, Value *val)
2104 {
2105 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2106
2107 if (tgsi.getSaturate()) {
2108 mkOp1(OP_SAT, dstTy, val, val);
2109 }
2110
2111 Value *ptr = NULL;
2112 if (dst.isIndirect(0))
2113 ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
2114
2115 if (info_out->io.genUserClip > 0 &&
2116 dst.getFile() == TGSI_FILE_OUTPUT &&
2117 !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
2118 mkMov(clipVtx[c], val);
2119 val = clipVtx[c];
2120 }
2121
2122 storeDst(dst, c, val, ptr);
2123 }
2124
2125 void
storeDst(const tgsi::Instruction::DstRegister dst,int c,Value * val,Value * ptr)2126 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
2127 Value *val, Value *ptr)
2128 {
2129 const unsigned f = dst.getFile();
2130 int idx = dst.getIndex(0);
2131 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2132
2133 if (f == TGSI_FILE_SYSTEM_VALUE) {
2134 assert(!ptr);
2135 mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
2136 } else
2137 if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
2138
2139 if (ptr || (info_out->out[idx].mask & (1 << c))) {
2140 /* Save the viewport index into a scratch register so that it can be
2141 exported at EMIT time */
2142 if (info_out->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
2143 prog->getType() == Program::TYPE_GEOMETRY &&
2144 viewport != NULL)
2145 mkOp1(OP_MOV, TYPE_U32, viewport, val);
2146 else
2147 mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
2148 info_out->out[idx].patch;
2149 }
2150 } else
2151 if (f == TGSI_FILE_TEMPORARY ||
2152 f == TGSI_FILE_ADDRESS ||
2153 f == TGSI_FILE_OUTPUT) {
2154 if (f == TGSI_FILE_TEMPORARY) {
2155 int arrayid = dst.getArrayId();
2156 if (!arrayid)
2157 arrayid = code->tempArrayId[idx];
2158 adjustTempIndex(arrayid, idx, idx2d);
2159 }
2160
2161 getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
2162 } else {
2163 assert(!"invalid dst file");
2164 }
2165 }
2166
2167 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
2168 for (chan = 0; chan < 4; ++chan) \
2169 if (!inst.getDst(d).isMasked(chan))
2170
2171 Value *
buildDot(int dim)2172 Converter::buildDot(int dim)
2173 {
2174 assert(dim > 0);
2175
2176 Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
2177 Value *dotp = getScratch();
2178
2179 mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1)
2180 ->dnz = info->io.mul_zero_wins;
2181
2182 for (int c = 1; c < dim; ++c) {
2183 src0 = fetchSrc(0, c);
2184 src1 = fetchSrc(1, c);
2185 mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp)
2186 ->dnz = info->io.mul_zero_wins;
2187 }
2188 return dotp;
2189 }
2190
2191 void
insertConvergenceOps(BasicBlock * conv,BasicBlock * fork)2192 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
2193 {
2194 FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
2195 join->fixed = 1;
2196 conv->insertHead(join);
2197
2198 assert(!fork->joinAt);
2199 fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
2200 fork->insertBefore(fork->getExit(), fork->joinAt);
2201 }
2202
2203 void
setTexRS(TexInstruction * tex,unsigned int & s,int R,int S)2204 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
2205 {
2206 unsigned rIdx = 0, sIdx = 0;
2207
2208 if (R >= 0 && tgsi.getSrc(R).getFile() != TGSI_FILE_SAMPLER) {
2209 // This is the bindless case. We have to get the actual value and pass
2210 // it in. This will be the complete handle.
2211 tex->tex.rIndirectSrc = s;
2212 tex->setSrc(s++, fetchSrc(R, 0));
2213 tex->setTexture(tgsi.getTexture(code, R), 0xff, 0x1f);
2214 tex->tex.bindless = true;
2215 return;
2216 }
2217
2218 if (R >= 0)
2219 rIdx = tgsi.getSrc(R).getIndex(0);
2220 if (S >= 0)
2221 sIdx = tgsi.getSrc(S).getIndex(0);
2222
2223 tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
2224
2225 if (tgsi.getSrc(R).isIndirect(0)) {
2226 tex->tex.rIndirectSrc = s;
2227 tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
2228 }
2229 if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
2230 tex->tex.sIndirectSrc = s;
2231 tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
2232 }
2233 }
2234
2235 void
handleTXQ(Value * dst0[4],enum TexQuery query,int R)2236 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
2237 {
2238 TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
2239 tex->tex.query = query;
2240 unsigned int c, d;
2241
2242 for (d = 0, c = 0; c < 4; ++c) {
2243 if (!dst0[c])
2244 continue;
2245 tex->tex.mask |= 1 << c;
2246 tex->setDef(d++, dst0[c]);
2247 }
2248 if (query == TXQ_DIMS)
2249 tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
2250 else
2251 tex->setSrc((c = 0), zero);
2252
2253 setTexRS(tex, ++c, R, -1);
2254
2255 bb->insertTail(tex);
2256 }
2257
2258 void
loadProjTexCoords(Value * dst[4],Value * src[4],unsigned int mask)2259 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
2260 {
2261 Value *proj = fetchSrc(0, 3);
2262 Instruction *insn = proj->getUniqueInsn();
2263 int c;
2264
2265 if (insn->op == OP_PINTERP) {
2266 bb->insertTail(insn = cloneForward(func, insn));
2267 insn->op = OP_LINTERP;
2268 insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
2269 insn->setSrc(1, NULL);
2270 proj = insn->getDef(0);
2271 }
2272 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
2273
2274 for (c = 0; c < 4; ++c) {
2275 if (!(mask & (1 << c)))
2276 continue;
2277 if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
2278 continue;
2279 mask &= ~(1 << c);
2280
2281 bb->insertTail(insn = cloneForward(func, insn));
2282 insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
2283 insn->setSrc(1, proj);
2284 dst[c] = insn->getDef(0);
2285 }
2286 if (!mask)
2287 return;
2288
2289 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
2290
2291 for (c = 0; c < 4; ++c)
2292 if (mask & (1 << c))
2293 dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
2294 }
2295
2296 // order of nv50 ir sources: x y z layer lod/bias shadow
2297 // order of TGSI TEX sources: x y z layer shadow lod/bias
2298 // lowering will finally set the hw specific order (like array first on nvc0)
2299 void
handleTEX(Value * dst[4],int R,int S,int L,int C,int Dx,int Dy)2300 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
2301 {
2302 Value *arg[4], *src[8];
2303 Value *lod = NULL, *shd = NULL;
2304 unsigned int s, c, d;
2305 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2306
2307 TexInstruction::Target tgt = tgsi.getTexture(code, R);
2308
2309 for (s = 0; s < tgt.getArgCount(); ++s)
2310 arg[s] = src[s] = fetchSrc(0, s);
2311
2312 if (tgsi.getOpcode() == TGSI_OPCODE_TEX_LZ)
2313 lod = loadImm(NULL, 0);
2314 else if (texi->op == OP_TXL || texi->op == OP_TXB)
2315 lod = fetchSrc(L >> 4, L & 3);
2316
2317 if (C == 0x0f)
2318 C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2319
2320 if (tgt == TEX_TARGET_CUBE_ARRAY_SHADOW) {
2321 switch (tgsi.getOpcode()) {
2322 case TGSI_OPCODE_TG4: shd = fetchSrc(1, 0); break;
2323 case TGSI_OPCODE_TEX2: shd = fetchSrc(1, 0); break;
2324 case TGSI_OPCODE_TXB2: shd = fetchSrc(1, 1); break;
2325 case TGSI_OPCODE_TXL2: shd = fetchSrc(1, 1); break;
2326 default: assert(!"unexpected opcode with cube array shadow"); break;
2327 }
2328 }
2329 else if (tgt.isShadow())
2330 shd = fetchSrc(C >> 4, C & 3);
2331
2332 if (texi->op == OP_TXD) {
2333 for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2334 texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2335 texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2336 }
2337 }
2338
2339 // cube textures don't care about projection value, it's divided out
2340 if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2341 unsigned int n = tgt.getDim();
2342 if (shd) {
2343 arg[n] = shd;
2344 ++n;
2345 assert(tgt.getDim() == tgt.getArgCount());
2346 }
2347 loadProjTexCoords(src, arg, (1 << n) - 1);
2348 if (shd)
2349 shd = src[n - 1];
2350 }
2351
2352 for (c = 0, d = 0; c < 4; ++c) {
2353 if (dst[c]) {
2354 texi->setDef(d++, dst[c]);
2355 texi->tex.mask |= 1 << c;
2356 } else {
2357 // NOTE: maybe hook up def too, for CSE
2358 }
2359 }
2360 for (s = 0; s < tgt.getArgCount(); ++s)
2361 texi->setSrc(s, src[s]);
2362 if (lod)
2363 texi->setSrc(s++, lod);
2364 if (shd)
2365 texi->setSrc(s++, shd);
2366
2367 setTexRS(texi, s, R, S);
2368
2369 if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2370 texi->tex.levelZero = true;
2371 if (prog->getType() != Program::TYPE_FRAGMENT &&
2372 (tgsi.getOpcode() == TGSI_OPCODE_TEX ||
2373 tgsi.getOpcode() == TGSI_OPCODE_TEX2 ||
2374 tgsi.getOpcode() == TGSI_OPCODE_TXP))
2375 texi->tex.levelZero = true;
2376 if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2377 texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, code->immd.data);
2378
2379 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2380 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2381 for (c = 0; c < 3; ++c) {
2382 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2383 texi->offset[s][c].setInsn(texi);
2384 }
2385 }
2386
2387 bb->insertTail(texi);
2388 }
2389
2390 // 1st source: xyz = coordinates, w = lod/sample
2391 // 2nd source: offset
2392 void
handleTXF(Value * dst[4],int R,int L_M)2393 Converter::handleTXF(Value *dst[4], int R, int L_M)
2394 {
2395 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2396 int ms;
2397 unsigned int c, d, s;
2398
2399 texi->tex.target = tgsi.getTexture(code, R);
2400
2401 ms = texi->tex.target.isMS() ? 1 : 0;
2402 texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2403
2404 for (c = 0, d = 0; c < 4; ++c) {
2405 if (dst[c]) {
2406 texi->setDef(d++, dst[c]);
2407 texi->tex.mask |= 1 << c;
2408 }
2409 }
2410 for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2411 texi->setSrc(c, fetchSrc(0, c));
2412 if (!ms && tgsi.getOpcode() == TGSI_OPCODE_TXF_LZ)
2413 texi->setSrc(c++, loadImm(NULL, 0));
2414 else
2415 texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2416
2417 setTexRS(texi, c, R, -1);
2418
2419 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2420 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2421 for (c = 0; c < 3; ++c) {
2422 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2423 texi->offset[s][c].setInsn(texi);
2424 }
2425 }
2426
2427 bb->insertTail(texi);
2428 }
2429
2430 void
handleFBFETCH(Value * dst[4])2431 Converter::handleFBFETCH(Value *dst[4])
2432 {
2433 TexInstruction *texi = new_TexInstruction(func, OP_TXF);
2434 unsigned int c, d;
2435
2436 texi->tex.target = TEX_TARGET_2D_MS_ARRAY;
2437 texi->tex.levelZero = 1;
2438 texi->tex.useOffsets = 0;
2439
2440 for (c = 0, d = 0; c < 4; ++c) {
2441 if (dst[c]) {
2442 texi->setDef(d++, dst[c]);
2443 texi->tex.mask |= 1 << c;
2444 }
2445 }
2446
2447 Value *x = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 0));
2448 Value *y = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 1));
2449 Value *z = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_LAYER, 0));
2450 Value *ms = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_SAMPLE_INDEX, 0));
2451
2452 mkCvt(OP_CVT, TYPE_U32, x, TYPE_F32, x)->rnd = ROUND_Z;
2453 mkCvt(OP_CVT, TYPE_U32, y, TYPE_F32, y)->rnd = ROUND_Z;
2454 texi->setSrc(0, x);
2455 texi->setSrc(1, y);
2456 texi->setSrc(2, z);
2457 texi->setSrc(3, ms);
2458
2459 texi->tex.r = texi->tex.s = -1;
2460
2461 bb->insertTail(texi);
2462 }
2463
2464 void
handleLIT(Value * dst0[4])2465 Converter::handleLIT(Value *dst0[4])
2466 {
2467 Value *val0 = NULL;
2468 unsigned int mask = tgsi.getDst(0).getMask();
2469
2470 if (mask & (1 << 0))
2471 loadImm(dst0[0], 1.0f);
2472
2473 if (mask & (1 << 3))
2474 loadImm(dst0[3], 1.0f);
2475
2476 if (mask & (3 << 1)) {
2477 val0 = getScratch();
2478 mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2479 if (mask & (1 << 1))
2480 mkMov(dst0[1], val0);
2481 }
2482
2483 if (mask & (1 << 2)) {
2484 Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2485 Value *val1 = getScratch(), *val3 = getScratch();
2486
2487 Value *pos128 = loadImm(NULL, +127.999999f);
2488 Value *neg128 = loadImm(NULL, -127.999999f);
2489
2490 mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2491 mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2492 mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2493 mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2494
2495 mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2496 }
2497 }
2498
2499 /* Keep this around for now as reference when adding img support
2500 static inline bool
2501 isResourceSpecial(const int r)
2502 {
2503 return (r == TGSI_RESOURCE_GLOBAL ||
2504 r == TGSI_RESOURCE_LOCAL ||
2505 r == TGSI_RESOURCE_PRIVATE ||
2506 r == TGSI_RESOURCE_INPUT);
2507 }
2508
2509 static inline bool
2510 isResourceRaw(const tgsi::Source *code, const int r)
2511 {
2512 return isResourceSpecial(r) || code->resources[r].raw;
2513 }
2514
2515 static inline nv50_ir::TexTarget
2516 getResourceTarget(const tgsi::Source *code, int r)
2517 {
2518 if (isResourceSpecial(r))
2519 return nv50_ir::TEX_TARGET_BUFFER;
2520 return tgsi::translateTexture(code->resources.at(r).target);
2521 }
2522
2523 Symbol *
2524 Converter::getResourceBase(const int r)
2525 {
2526 Symbol *sym = NULL;
2527
2528 switch (r) {
2529 case TGSI_RESOURCE_GLOBAL:
2530 sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL,
2531 info->io.auxCBSlot);
2532 break;
2533 case TGSI_RESOURCE_LOCAL:
2534 assert(prog->getType() == Program::TYPE_COMPUTE);
2535 sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2536 info->prop.cp.sharedOffset);
2537 break;
2538 case TGSI_RESOURCE_PRIVATE:
2539 sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2540 info->bin.tlsSpace);
2541 break;
2542 case TGSI_RESOURCE_INPUT:
2543 assert(prog->getType() == Program::TYPE_COMPUTE);
2544 sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2545 info->prop.cp.inputOffset);
2546 break;
2547 default:
2548 sym = new_Symbol(prog,
2549 nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2550 break;
2551 }
2552 return sym;
2553 }
2554
2555 void
2556 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2557 {
2558 const int arg =
2559 TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2560
2561 for (int c = 0; c < arg; ++c)
2562 coords.push_back(fetchSrc(s, c));
2563
2564 // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2565 if (r == TGSI_RESOURCE_LOCAL ||
2566 r == TGSI_RESOURCE_PRIVATE ||
2567 r == TGSI_RESOURCE_INPUT)
2568 coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2569 coords[0]);
2570 }
2571
2572 static inline int
2573 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2574 {
2575 int n = 0;
2576
2577 while (mask) {
2578 if (mask & 1) {
2579 size[n]++;
2580 } else {
2581 if (size[n])
2582 comp[n = 1] = size[0] + 1;
2583 else
2584 comp[n]++;
2585 }
2586 mask >>= 1;
2587 }
2588 if (size[0] == 3) {
2589 n = 1;
2590 size[0] = (comp[0] == 1) ? 1 : 2;
2591 size[1] = 3 - size[0];
2592 comp[1] = comp[0] + size[0];
2593 }
2594 return n + 1;
2595 }
2596 */
2597 void
getImageCoords(std::vector<Value * > & coords,int s)2598 Converter::getImageCoords(std::vector<Value *> &coords, int s)
2599 {
2600 TexInstruction::Target t =
2601 TexInstruction::Target(tgsi.getImageTarget());
2602 const int arg = t.getDim() + (t.isArray() || t.isCube());
2603
2604 for (int c = 0; c < arg; ++c)
2605 coords.push_back(fetchSrc(s, c));
2606
2607 if (t.isMS())
2608 coords.push_back(fetchSrc(s, 3));
2609 }
2610
2611 // For raw loads, granularity is 4 byte.
2612 // Usage of the texture read mask on OP_SULDP is not allowed.
2613 void
handleLOAD(Value * dst0[4])2614 Converter::handleLOAD(Value *dst0[4])
2615 {
2616 const int r = tgsi.getSrc(0).getIndex(0);
2617 int c;
2618 std::vector<Value *> off, src, ldv, def;
2619 Value *ind = NULL;
2620
2621 if (tgsi.getSrc(0).isIndirect(0))
2622 ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2623
2624 switch (tgsi.getSrc(0).getFile()) {
2625 case TGSI_FILE_BUFFER:
2626 case TGSI_FILE_MEMORY:
2627 for (c = 0; c < 4; ++c) {
2628 if (!dst0[c])
2629 continue;
2630
2631 Value *off;
2632 Symbol *sym;
2633 uint32_t src0_component_offset = tgsi.getSrc(0).getSwizzle(c) * 4;
2634
2635 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2636 off = NULL;
2637 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2638 tgsi.getSrc(1).getValueU32(0, code->immd.data) +
2639 src0_component_offset);
2640 } else {
2641 // yzw are ignored for buffers
2642 off = fetchSrc(1, 0);
2643 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2644 src0_component_offset);
2645 }
2646
2647 Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2648 if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER &&
2649 code->bufferAtomics[r])
2650 ld->cache = nv50_ir::CACHE_CG;
2651 else
2652 ld->cache = tgsi.getCacheMode();
2653 if (ind)
2654 ld->setIndirect(0, 1, ind);
2655 }
2656 break;
2657 default: {
2658 getImageCoords(off, 1);
2659 def.resize(4);
2660
2661 for (c = 0; c < 4; ++c) {
2662 if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2663 def[c] = getScratch();
2664 else
2665 def[c] = dst0[c];
2666 }
2667
2668 bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2669 if (bindless)
2670 ind = fetchSrc(0, 0);
2671
2672 TexInstruction *ld =
2673 mkTex(OP_SULDP, tgsi.getImageTarget(), 0, 0, def, off);
2674 ld->tex.mask = tgsi.getDst(0).getMask();
2675 ld->tex.format = tgsi.getImageFormat();
2676 ld->cache = tgsi.getCacheMode();
2677 ld->tex.bindless = bindless;
2678 if (!bindless)
2679 ld->tex.r = r;
2680 if (ind)
2681 ld->setIndirectR(ind);
2682
2683 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2684 if (dst0[c] != def[c])
2685 mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2686 break;
2687 }
2688 }
2689
2690
2691 /* Keep this around for now as reference when adding img support
2692 getResourceCoords(off, r, 1);
2693
2694 if (isResourceRaw(code, r)) {
2695 uint8_t mask = 0;
2696 uint8_t comp[2] = { 0, 0 };
2697 uint8_t size[2] = { 0, 0 };
2698
2699 Symbol *base = getResourceBase(r);
2700
2701 // determine the base and size of the at most 2 load ops
2702 for (c = 0; c < 4; ++c)
2703 if (!tgsi.getDst(0).isMasked(c))
2704 mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2705
2706 int n = partitionLoadStore(comp, size, mask);
2707
2708 src = off;
2709
2710 def.resize(4); // index by component, the ones we need will be non-NULL
2711 for (c = 0; c < 4; ++c) {
2712 if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2713 def[c] = dst0[c];
2714 else
2715 if (mask & (1 << c))
2716 def[c] = getScratch();
2717 }
2718
2719 const bool useLd = isResourceSpecial(r) ||
2720 (info->io.nv50styleSurfaces &&
2721 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2722
2723 for (int i = 0; i < n; ++i) {
2724 ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2725
2726 if (comp[i]) // adjust x component of source address if necessary
2727 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2728 off[0], mkImm(comp[i] * 4));
2729 else
2730 src[0] = off[0];
2731
2732 if (useLd) {
2733 Instruction *ld =
2734 mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2735 for (size_t c = 1; c < ldv.size(); ++c)
2736 ld->setDef(c, ldv[c]);
2737 } else {
2738 mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2739 0, ldv, src)->dType = typeOfSize(size[i] * 4);
2740 }
2741 }
2742 } else {
2743 def.resize(4);
2744 for (c = 0; c < 4; ++c) {
2745 if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2746 def[c] = getScratch();
2747 else
2748 def[c] = dst0[c];
2749 }
2750
2751 mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2752 def, off);
2753 }
2754 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2755 if (dst0[c] != def[c])
2756 mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2757 */
2758 }
2759
2760 // For formatted stores, the write mask on OP_SUSTP can be used.
2761 // Raw stores have to be split.
2762 void
handleSTORE()2763 Converter::handleSTORE()
2764 {
2765 const int r = tgsi.getDst(0).getIndex(0);
2766 int c;
2767 std::vector<Value *> off, src, dummy;
2768 Value *ind = NULL;
2769
2770 if (tgsi.getDst(0).isIndirect(0))
2771 ind = fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0);
2772
2773 switch (tgsi.getDst(0).getFile()) {
2774 case TGSI_FILE_BUFFER:
2775 case TGSI_FILE_MEMORY:
2776 for (c = 0; c < 4; ++c) {
2777 if (!(tgsi.getDst(0).getMask() & (1 << c)))
2778 continue;
2779
2780 Symbol *sym;
2781 Value *off;
2782 if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2783 off = NULL;
2784 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2785 tgsi.getSrc(0).getValueU32(0, code->immd.data) + 4 * c);
2786 } else {
2787 // yzw are ignored for buffers
2788 off = fetchSrc(0, 0);
2789 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2790 }
2791
2792 Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2793 st->cache = tgsi.getCacheMode();
2794 if (ind)
2795 st->setIndirect(0, 1, ind);
2796 }
2797 break;
2798 default: {
2799 getImageCoords(off, 0);
2800 src = off;
2801
2802 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2803 src.push_back(fetchSrc(1, c));
2804
2805 bool bindless = tgsi.getDst(0).getFile() != TGSI_FILE_IMAGE;
2806 if (bindless)
2807 ind = fetchDst(0, 0);
2808
2809 TexInstruction *st =
2810 mkTex(OP_SUSTP, tgsi.getImageTarget(), 0, 0, dummy, src);
2811 st->tex.mask = tgsi.getDst(0).getMask();
2812 st->tex.format = tgsi.getImageFormat();
2813 st->cache = tgsi.getCacheMode();
2814 st->tex.bindless = bindless;
2815 if (!bindless)
2816 st->tex.r = r;
2817 if (ind)
2818 st->setIndirectR(ind);
2819
2820 break;
2821 }
2822 }
2823
2824 /* Keep this around for now as reference when adding img support
2825 getResourceCoords(off, r, 0);
2826 src = off;
2827 const int s = src.size();
2828
2829 if (isResourceRaw(code, r)) {
2830 uint8_t comp[2] = { 0, 0 };
2831 uint8_t size[2] = { 0, 0 };
2832
2833 int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2834
2835 Symbol *base = getResourceBase(r);
2836
2837 const bool useSt = isResourceSpecial(r) ||
2838 (info->io.nv50styleSurfaces &&
2839 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2840
2841 for (int i = 0; i < n; ++i) {
2842 if (comp[i]) // adjust x component of source address if necessary
2843 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2844 off[0], mkImm(comp[i] * 4));
2845 else
2846 src[0] = off[0];
2847
2848 const DataType stTy = typeOfSize(size[i] * 4);
2849
2850 if (useSt) {
2851 Instruction *st =
2852 mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2853 for (c = 1; c < size[i]; ++c)
2854 st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2855 st->setIndirect(0, 0, src[0]);
2856 } else {
2857 // attach values to be stored
2858 src.resize(s + size[i]);
2859 for (c = 0; c < size[i]; ++c)
2860 src[s + c] = fetchSrc(1, comp[i] + c);
2861 mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2862 0, dummy, src)->setType(stTy);
2863 }
2864 }
2865 } else {
2866 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2867 src.push_back(fetchSrc(1, c));
2868
2869 mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2870 dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2871 }
2872 */
2873 }
2874
2875 // XXX: These only work on resources with the single-component u32/s32 formats.
2876 // Therefore the result is replicated. This might not be intended by TGSI, but
2877 // operating on more than 1 component would produce undefined results because
2878 // they do not exist.
2879 void
handleATOM(Value * dst0[4],DataType ty,uint16_t subOp)2880 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2881 {
2882 const int r = tgsi.getSrc(0).getIndex(0);
2883 std::vector<Value *> srcv;
2884 std::vector<Value *> defv;
2885 LValue *dst = getScratch();
2886 Value *ind = NULL;
2887
2888 if (tgsi.getSrc(0).isIndirect(0))
2889 ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2890
2891 switch (tgsi.getSrc(0).getFile()) {
2892 case TGSI_FILE_BUFFER:
2893 case TGSI_FILE_MEMORY:
2894 for (int c = 0; c < 4; ++c) {
2895 if (!dst0[c])
2896 continue;
2897
2898 Instruction *insn;
2899 Value *off = fetchSrc(1, c);
2900 Value *sym;
2901 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2902 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2903 tgsi.getSrc(1).getValueU32(c, code->immd.data));
2904 else
2905 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2906 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2907 insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2908 else
2909 insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2910 if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2911 insn->setIndirect(0, 0, off);
2912 if (ind)
2913 insn->setIndirect(0, 1, ind);
2914 insn->subOp = subOp;
2915 }
2916 for (int c = 0; c < 4; ++c)
2917 if (dst0[c])
2918 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2919 break;
2920 default: {
2921 getImageCoords(srcv, 1);
2922 defv.push_back(dst);
2923 srcv.push_back(fetchSrc(2, 0));
2924
2925 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2926 srcv.push_back(fetchSrc(3, 0));
2927
2928 bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2929 if (bindless)
2930 ind = fetchSrc(0, 0);
2931
2932 TexInstruction *tex = mkTex(OP_SUREDP, tgsi.getImageTarget(),
2933 0, 0, defv, srcv);
2934 tex->subOp = subOp;
2935 tex->tex.mask = 1;
2936 tex->tex.format = tgsi.getImageFormat();
2937 tex->setType(ty);
2938 tex->tex.bindless = bindless;
2939 if (!bindless)
2940 tex->tex.r = r;
2941 if (ind)
2942 tex->setIndirectR(ind);
2943
2944 for (int c = 0; c < 4; ++c)
2945 if (dst0[c])
2946 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2947 break;
2948 }
2949 }
2950
2951 /* Keep this around for now as reference when adding img support
2952 getResourceCoords(srcv, r, 1);
2953
2954 if (isResourceSpecial(r)) {
2955 assert(r != TGSI_RESOURCE_INPUT);
2956 Instruction *insn;
2957 insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2958 insn->subOp = subOp;
2959 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2960 insn->setSrc(2, fetchSrc(3, 0));
2961 insn->setIndirect(0, 0, srcv.at(0));
2962 } else {
2963 operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2964 TexTarget targ = getResourceTarget(code, r);
2965 int idx = code->resources[r].slot;
2966 defv.push_back(dst);
2967 srcv.push_back(fetchSrc(2, 0));
2968 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2969 srcv.push_back(fetchSrc(3, 0));
2970 TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2971 tex->subOp = subOp;
2972 tex->tex.mask = 1;
2973 tex->setType(ty);
2974 }
2975
2976 for (int c = 0; c < 4; ++c)
2977 if (dst0[c])
2978 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2979 */
2980 }
2981
2982 void
handleINTERP(Value * dst[4])2983 Converter::handleINTERP(Value *dst[4])
2984 {
2985 // Check whether the input is linear. All other attributes ignored.
2986 Instruction *insn;
2987 Value *offset = NULL, *ptr = NULL, *w = NULL;
2988 Symbol *sym[4] = { NULL };
2989 bool linear;
2990 operation op = OP_NOP;
2991 int c, mode = 0;
2992
2993 tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2994
2995 // In some odd cases, in large part due to varying packing, the source
2996 // might not actually be an input. This is illegal TGSI, but it's easier to
2997 // account for it here than it is to fix it where the TGSI is being
2998 // generated. In that case, it's going to be a straight up mov (or sequence
2999 // of mov's) from the input in question. We follow the mov chain to see
3000 // which input we need to use.
3001 if (src.getFile() != TGSI_FILE_INPUT) {
3002 if (src.isIndirect(0)) {
3003 ERROR("Ignoring indirect input interpolation\n");
3004 return;
3005 }
3006 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3007 Value *val = fetchSrc(0, c);
3008 assert(val->defs.size() == 1);
3009 insn = val->getInsn();
3010 while (insn->op == OP_MOV) {
3011 assert(insn->getSrc(0)->defs.size() == 1);
3012 insn = insn->getSrc(0)->getInsn();
3013 if (!insn) {
3014 ERROR("Miscompiling shader due to unhandled INTERP\n");
3015 return;
3016 }
3017 }
3018 if (insn->op != OP_LINTERP && insn->op != OP_PINTERP) {
3019 ERROR("Trying to interpolate non-input, this is not allowed.\n");
3020 return;
3021 }
3022 sym[c] = insn->getSrc(0)->asSym();
3023 assert(sym[c]);
3024 op = insn->op;
3025 mode = insn->ipa;
3026 ptr = insn->getIndirect(0, 0);
3027 }
3028 } else {
3029 if (src.isIndirect(0))
3030 ptr = shiftAddress(fetchSrc(src.getIndirect(0), 0, NULL));
3031
3032 // We can assume that the fixed index will point to an input of the same
3033 // interpolation type in case of an indirect.
3034 // TODO: Make use of ArrayID.
3035 linear = info_out->in[src.getIndex(0)].linear;
3036 if (linear) {
3037 op = OP_LINTERP;
3038 mode = NV50_IR_INTERP_LINEAR;
3039 } else {
3040 op = OP_PINTERP;
3041 mode = NV50_IR_INTERP_PERSPECTIVE;
3042 }
3043 }
3044
3045 switch (tgsi.getOpcode()) {
3046 case TGSI_OPCODE_INTERP_CENTROID:
3047 mode |= NV50_IR_INTERP_CENTROID;
3048 break;
3049 case TGSI_OPCODE_INTERP_SAMPLE:
3050 insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
3051 insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
3052 mode |= NV50_IR_INTERP_OFFSET;
3053 break;
3054 case TGSI_OPCODE_INTERP_OFFSET: {
3055 // The input in src1.xy is float, but we need a single 32-bit value
3056 // where the upper and lower 16 bits are encoded in S0.12 format. We need
3057 // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
3058 // and then convert to s32.
3059 Value *offs[2];
3060 for (c = 0; c < 2; c++) {
3061 offs[c] = getScratch();
3062 mkOp2(OP_MIN, TYPE_F32, offs[c], fetchSrc(1, c), loadImm(NULL, 0.4375f));
3063 mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
3064 mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
3065 mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
3066 }
3067 offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
3068 offs[1], mkImm(0x1010), offs[0]);
3069 mode |= NV50_IR_INTERP_OFFSET;
3070 break;
3071 }
3072 }
3073
3074 if (op == OP_PINTERP) {
3075 if (offset) {
3076 w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
3077 mkOp1(OP_RCP, TYPE_F32, w, w);
3078 } else {
3079 w = fragCoord[3];
3080 }
3081 }
3082
3083
3084 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3085 insn = mkOp1(op, TYPE_F32, dst[c], sym[c] ? sym[c] : srcToSym(src, c));
3086 if (op == OP_PINTERP)
3087 insn->setSrc(1, w);
3088 if (offset)
3089 insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
3090 if (ptr)
3091 insn->setIndirect(0, 0, ptr);
3092
3093 insn->setInterpolate(mode);
3094 }
3095 }
3096
3097 bool
isEndOfSubroutine(uint ip)3098 Converter::isEndOfSubroutine(uint ip)
3099 {
3100 assert(ip < code->scan.num_instructions);
3101 tgsi::Instruction insn(&code->insns[ip]);
3102 return (insn.getOpcode() == TGSI_OPCODE_END ||
3103 insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
3104 // does END occur at end of main or the very end ?
3105 insn.getOpcode() == TGSI_OPCODE_BGNSUB);
3106 }
3107
3108 bool
handleInstruction(const struct tgsi_full_instruction * insn)3109 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
3110 {
3111 Instruction *geni;
3112
3113 Value *dst0[4], *rDst0[4];
3114 Value *src0, *src1, *src2, *src3;
3115 Value *val0, *val1;
3116 int c;
3117
3118 tgsi = tgsi::Instruction(insn);
3119
3120 bool useScratchDst = tgsi.checkDstSrcAliasing();
3121
3122 operation op = tgsi.getOP();
3123 dstTy = tgsi.inferDstType();
3124 srcTy = tgsi.inferSrcType();
3125
3126 unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
3127
3128 if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
3129 for (c = 0; c < 4; ++c) {
3130 rDst0[c] = acquireDst(0, c);
3131 dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
3132 }
3133 }
3134
3135 switch (tgsi.getOpcode()) {
3136 case TGSI_OPCODE_ADD:
3137 case TGSI_OPCODE_UADD:
3138 case TGSI_OPCODE_AND:
3139 case TGSI_OPCODE_DIV:
3140 case TGSI_OPCODE_IDIV:
3141 case TGSI_OPCODE_UDIV:
3142 case TGSI_OPCODE_MAX:
3143 case TGSI_OPCODE_MIN:
3144 case TGSI_OPCODE_IMAX:
3145 case TGSI_OPCODE_IMIN:
3146 case TGSI_OPCODE_UMAX:
3147 case TGSI_OPCODE_UMIN:
3148 case TGSI_OPCODE_MOD:
3149 case TGSI_OPCODE_UMOD:
3150 case TGSI_OPCODE_MUL:
3151 case TGSI_OPCODE_UMUL:
3152 case TGSI_OPCODE_IMUL_HI:
3153 case TGSI_OPCODE_UMUL_HI:
3154 case TGSI_OPCODE_OR:
3155 case TGSI_OPCODE_SHL:
3156 case TGSI_OPCODE_ISHR:
3157 case TGSI_OPCODE_USHR:
3158 case TGSI_OPCODE_XOR:
3159 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3160 src0 = fetchSrc(0, c);
3161 src1 = fetchSrc(1, c);
3162 geni = mkOp2(op, dstTy, dst0[c], src0, src1);
3163 geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3164 if (op == OP_MUL && dstTy == TYPE_F32)
3165 geni->dnz = info->io.mul_zero_wins;
3166 geni->precise = insn->Instruction.Precise;
3167 }
3168 break;
3169 case TGSI_OPCODE_MAD:
3170 case TGSI_OPCODE_UMAD:
3171 case TGSI_OPCODE_FMA:
3172 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3173 src0 = fetchSrc(0, c);
3174 src1 = fetchSrc(1, c);
3175 src2 = fetchSrc(2, c);
3176 geni = mkOp3(op, dstTy, dst0[c], src0, src1, src2);
3177 if (dstTy == TYPE_F32)
3178 geni->dnz = info->io.mul_zero_wins;
3179 geni->precise = insn->Instruction.Precise;
3180 }
3181 break;
3182 case TGSI_OPCODE_MOV:
3183 case TGSI_OPCODE_CEIL:
3184 case TGSI_OPCODE_FLR:
3185 case TGSI_OPCODE_TRUNC:
3186 case TGSI_OPCODE_RCP:
3187 case TGSI_OPCODE_SQRT:
3188 case TGSI_OPCODE_IABS:
3189 case TGSI_OPCODE_INEG:
3190 case TGSI_OPCODE_NOT:
3191 case TGSI_OPCODE_DDX:
3192 case TGSI_OPCODE_DDY:
3193 case TGSI_OPCODE_DDX_FINE:
3194 case TGSI_OPCODE_DDY_FINE:
3195 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3196 mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
3197 break;
3198 case TGSI_OPCODE_RSQ:
3199 src0 = fetchSrc(0, 0);
3200 val0 = getScratch();
3201 mkOp1(OP_ABS, TYPE_F32, val0, src0);
3202 mkOp1(OP_RSQ, TYPE_F32, val0, val0);
3203 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3204 mkMov(dst0[c], val0);
3205 break;
3206 case TGSI_OPCODE_ARL:
3207 case TGSI_OPCODE_ARR:
3208 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3209 const RoundMode rnd =
3210 tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
3211 src0 = fetchSrc(0, c);
3212 mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
3213 }
3214 break;
3215 case TGSI_OPCODE_UARL:
3216 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3217 mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
3218 break;
3219 case TGSI_OPCODE_POW:
3220 val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
3221 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3222 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3223 break;
3224 case TGSI_OPCODE_EX2:
3225 case TGSI_OPCODE_LG2:
3226 val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
3227 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3228 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3229 break;
3230 case TGSI_OPCODE_COS:
3231 case TGSI_OPCODE_SIN:
3232 val0 = getScratch();
3233 if (mask & 7) {
3234 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
3235 mkOp1(op, TYPE_F32, val0, val0);
3236 for (c = 0; c < 3; ++c)
3237 if (dst0[c])
3238 mkMov(dst0[c], val0);
3239 }
3240 if (dst0[3]) {
3241 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
3242 mkOp1(op, TYPE_F32, dst0[3], val0);
3243 }
3244 break;
3245 case TGSI_OPCODE_EXP:
3246 src0 = fetchSrc(0, 0);
3247 val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
3248 if (dst0[1])
3249 mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
3250 if (dst0[0])
3251 mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
3252 if (dst0[2])
3253 mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
3254 if (dst0[3])
3255 loadImm(dst0[3], 1.0f);
3256 break;
3257 case TGSI_OPCODE_LOG:
3258 src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
3259 val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
3260 if (dst0[0] || dst0[1])
3261 val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
3262 if (dst0[1]) {
3263 mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
3264 mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
3265 mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0)
3266 ->dnz = info->io.mul_zero_wins;
3267 }
3268 if (dst0[3])
3269 loadImm(dst0[3], 1.0f);
3270 break;
3271 case TGSI_OPCODE_DP2:
3272 val0 = buildDot(2);
3273 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3274 mkMov(dst0[c], val0);
3275 break;
3276 case TGSI_OPCODE_DP3:
3277 val0 = buildDot(3);
3278 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3279 mkMov(dst0[c], val0);
3280 break;
3281 case TGSI_OPCODE_DP4:
3282 val0 = buildDot(4);
3283 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3284 mkMov(dst0[c], val0);
3285 break;
3286 case TGSI_OPCODE_DST:
3287 if (dst0[0])
3288 loadImm(dst0[0], 1.0f);
3289 if (dst0[1]) {
3290 src0 = fetchSrc(0, 1);
3291 src1 = fetchSrc(1, 1);
3292 mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1)
3293 ->dnz = info->io.mul_zero_wins;
3294 }
3295 if (dst0[2])
3296 mkMov(dst0[2], fetchSrc(0, 2));
3297 if (dst0[3])
3298 mkMov(dst0[3], fetchSrc(1, 3));
3299 break;
3300 case TGSI_OPCODE_LRP:
3301 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3302 src0 = fetchSrc(0, c);
3303 src1 = fetchSrc(1, c);
3304 src2 = fetchSrc(2, c);
3305 mkOp3(OP_MAD, TYPE_F32, dst0[c],
3306 mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2)
3307 ->dnz = info->io.mul_zero_wins;
3308 }
3309 break;
3310 case TGSI_OPCODE_LIT:
3311 handleLIT(dst0);
3312 break;
3313 case TGSI_OPCODE_ISSG:
3314 case TGSI_OPCODE_SSG:
3315 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3316 src0 = fetchSrc(0, c);
3317 val0 = getScratch();
3318 val1 = getScratch();
3319 mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
3320 mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
3321 if (srcTy == TYPE_F32)
3322 mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
3323 else
3324 mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
3325 }
3326 break;
3327 case TGSI_OPCODE_UCMP:
3328 srcTy = TYPE_U32;
3329 /* fallthrough */
3330 case TGSI_OPCODE_CMP:
3331 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3332 src0 = fetchSrc(0, c);
3333 src1 = fetchSrc(1, c);
3334 src2 = fetchSrc(2, c);
3335 if (src1 == src2)
3336 mkMov(dst0[c], src1);
3337 else
3338 mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
3339 srcTy, dst0[c], srcTy, src1, src2, src0);
3340 }
3341 break;
3342 case TGSI_OPCODE_FRC:
3343 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3344 src0 = fetchSrc(0, c);
3345 val0 = getScratch();
3346 mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
3347 mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
3348 }
3349 break;
3350 case TGSI_OPCODE_ROUND:
3351 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3352 mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
3353 ->rnd = ROUND_NI;
3354 break;
3355 case TGSI_OPCODE_SLT:
3356 case TGSI_OPCODE_SGE:
3357 case TGSI_OPCODE_SEQ:
3358 case TGSI_OPCODE_SGT:
3359 case TGSI_OPCODE_SLE:
3360 case TGSI_OPCODE_SNE:
3361 case TGSI_OPCODE_FSEQ:
3362 case TGSI_OPCODE_FSGE:
3363 case TGSI_OPCODE_FSLT:
3364 case TGSI_OPCODE_FSNE:
3365 case TGSI_OPCODE_ISGE:
3366 case TGSI_OPCODE_ISLT:
3367 case TGSI_OPCODE_USEQ:
3368 case TGSI_OPCODE_USGE:
3369 case TGSI_OPCODE_USLT:
3370 case TGSI_OPCODE_USNE:
3371 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3372 src0 = fetchSrc(0, c);
3373 src1 = fetchSrc(1, c);
3374 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3375 }
3376 break;
3377 case TGSI_OPCODE_VOTE_ALL:
3378 case TGSI_OPCODE_VOTE_ANY:
3379 case TGSI_OPCODE_VOTE_EQ:
3380 val0 = new_LValue(func, FILE_PREDICATE);
3381 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3382 mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, c), zero);
3383 mkOp1(op, dstTy, val0, val0)
3384 ->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3385 mkCvt(OP_CVT, TYPE_U32, dst0[c], TYPE_U8, val0);
3386 }
3387 break;
3388 case TGSI_OPCODE_BALLOT:
3389 if (!tgsi.getDst(0).isMasked(0)) {
3390 val0 = new_LValue(func, FILE_PREDICATE);
3391 mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, 0), zero);
3392 mkOp1(op, TYPE_U32, dst0[0], val0)->subOp = NV50_IR_SUBOP_VOTE_ANY;
3393 }
3394 if (!tgsi.getDst(0).isMasked(1))
3395 mkMov(dst0[1], zero, TYPE_U32);
3396 break;
3397 case TGSI_OPCODE_READ_FIRST:
3398 // ReadFirstInvocationARB(src) is implemented as
3399 // ReadInvocationARB(src, findLSB(ballot(true)))
3400 val0 = getScratch();
3401 mkOp1(OP_VOTE, TYPE_U32, val0, mkImm(1))->subOp = NV50_IR_SUBOP_VOTE_ANY;
3402 mkOp1(OP_BREV, TYPE_U32, val0, val0);
3403 mkOp1(OP_BFIND, TYPE_U32, val0, val0)->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3404 src1 = val0;
3405 /* fallthrough */
3406 case TGSI_OPCODE_READ_INVOC:
3407 if (tgsi.getOpcode() == TGSI_OPCODE_READ_INVOC)
3408 src1 = fetchSrc(1, 0);
3409 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3410 geni = mkOp3(op, dstTy, dst0[c], fetchSrc(0, c), src1, mkImm(0x1f));
3411 geni->subOp = NV50_IR_SUBOP_SHFL_IDX;
3412 }
3413 break;
3414 case TGSI_OPCODE_CLOCK:
3415 // Stick the 32-bit clock into the high dword of the logical result.
3416 if (!tgsi.getDst(0).isMasked(0))
3417 mkOp1(OP_MOV, TYPE_U32, dst0[0], zero);
3418 if (!tgsi.getDst(0).isMasked(1))
3419 mkOp1(OP_RDSV, TYPE_U32, dst0[1], mkSysVal(SV_CLOCK, 0))->fixed = 1;
3420 break;
3421 case TGSI_OPCODE_READ_HELPER:
3422 if (!tgsi.getDst(0).isMasked(0))
3423 mkOp1(OP_RDSV, TYPE_U32, dst0[0], mkSysVal(SV_THREAD_KILL, 0))
3424 ->fixed = 1;
3425 break;
3426 case TGSI_OPCODE_KILL_IF:
3427 val0 = new_LValue(func, FILE_PREDICATE);
3428 mask = 0;
3429 for (c = 0; c < 4; ++c) {
3430 const int s = tgsi.getSrc(0).getSwizzle(c);
3431 if (mask & (1 << s))
3432 continue;
3433 mask |= 1 << s;
3434 mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
3435 mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
3436 }
3437 break;
3438 case TGSI_OPCODE_KILL:
3439 case TGSI_OPCODE_DEMOTE:
3440 // TODO: Should we make KILL exit that invocation? Some old shaders
3441 // don't like that.
3442 mkOp(OP_DISCARD, TYPE_NONE, NULL);
3443 break;
3444 case TGSI_OPCODE_TEX:
3445 case TGSI_OPCODE_TEX_LZ:
3446 case TGSI_OPCODE_TXB:
3447 case TGSI_OPCODE_TXL:
3448 case TGSI_OPCODE_TXP:
3449 case TGSI_OPCODE_LODQ:
3450 // R S L C Dx Dy
3451 handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
3452 break;
3453 case TGSI_OPCODE_TXD:
3454 handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
3455 break;
3456 case TGSI_OPCODE_TG4:
3457 handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
3458 break;
3459 case TGSI_OPCODE_TEX2:
3460 handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
3461 break;
3462 case TGSI_OPCODE_TXB2:
3463 case TGSI_OPCODE_TXL2:
3464 handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
3465 break;
3466 case TGSI_OPCODE_SAMPLE:
3467 case TGSI_OPCODE_SAMPLE_B:
3468 case TGSI_OPCODE_SAMPLE_D:
3469 case TGSI_OPCODE_SAMPLE_L:
3470 case TGSI_OPCODE_SAMPLE_C:
3471 case TGSI_OPCODE_SAMPLE_C_LZ:
3472 handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
3473 break;
3474 case TGSI_OPCODE_TXF_LZ:
3475 case TGSI_OPCODE_TXF:
3476 handleTXF(dst0, 1, 0x03);
3477 break;
3478 case TGSI_OPCODE_SAMPLE_I:
3479 handleTXF(dst0, 1, 0x03);
3480 break;
3481 case TGSI_OPCODE_SAMPLE_I_MS:
3482 handleTXF(dst0, 1, 0x20);
3483 break;
3484 case TGSI_OPCODE_TXQ:
3485 case TGSI_OPCODE_SVIEWINFO:
3486 handleTXQ(dst0, TXQ_DIMS, 1);
3487 break;
3488 case TGSI_OPCODE_TXQS:
3489 // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
3490 // be in .x
3491 dst0[1] = dst0[2] = dst0[3] = NULL;
3492 std::swap(dst0[0], dst0[2]);
3493 handleTXQ(dst0, TXQ_TYPE, 0);
3494 std::swap(dst0[0], dst0[2]);
3495 break;
3496 case TGSI_OPCODE_FBFETCH:
3497 handleFBFETCH(dst0);
3498 break;
3499 case TGSI_OPCODE_F2I:
3500 case TGSI_OPCODE_F2U:
3501 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3502 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
3503 break;
3504 case TGSI_OPCODE_I2F:
3505 case TGSI_OPCODE_U2F:
3506 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3507 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
3508 break;
3509 case TGSI_OPCODE_PK2H:
3510 val0 = getScratch();
3511 val1 = getScratch();
3512 mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
3513 mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
3514 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3515 mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
3516 break;
3517 case TGSI_OPCODE_UP2H:
3518 src0 = fetchSrc(0, 0);
3519 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3520 geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3521 geni->subOp = c & 1;
3522 }
3523 break;
3524 case TGSI_OPCODE_EMIT:
3525 /* export the saved viewport index */
3526 if (viewport != NULL) {
3527 Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3528 info_out->out[info->io.viewportId].slot[0] * 4);
3529 mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3530 }
3531 /* handle user clip planes for each emitted vertex */
3532 if (info_out->io.genUserClip > 0)
3533 handleUserClipPlanes();
3534 /* fallthrough */
3535 case TGSI_OPCODE_ENDPRIM:
3536 {
3537 // get vertex stream (must be immediate)
3538 unsigned int stream = tgsi.getSrc(0).getValueU32(0, code->immd.data);
3539 if (stream && op == OP_RESTART)
3540 break;
3541 if (info_out->prop.gp.maxVertices == 0)
3542 break;
3543 src0 = mkImm(stream);
3544 mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3545 break;
3546 }
3547 case TGSI_OPCODE_IF:
3548 case TGSI_OPCODE_UIF:
3549 {
3550 BasicBlock *ifBB = new BasicBlock(func);
3551
3552 bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3553 condBBs.push(bb);
3554 joinBBs.push(bb);
3555
3556 mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3557
3558 setPosition(ifBB, true);
3559 }
3560 break;
3561 case TGSI_OPCODE_ELSE:
3562 {
3563 BasicBlock *elseBB = new BasicBlock(func);
3564 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3565
3566 forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3567 condBBs.push(bb);
3568
3569 forkBB->getExit()->asFlow()->target.bb = elseBB;
3570 if (!bb->isTerminated())
3571 mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3572
3573 setPosition(elseBB, true);
3574 }
3575 break;
3576 case TGSI_OPCODE_ENDIF:
3577 {
3578 BasicBlock *convBB = new BasicBlock(func);
3579 BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3580 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3581
3582 if (!bb->isTerminated()) {
3583 // we only want join if none of the clauses ended with CONT/BREAK/RET
3584 if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3585 insertConvergenceOps(convBB, forkBB);
3586 mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3587 bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3588 }
3589
3590 if (prevBB->getExit()->op == OP_BRA) {
3591 prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3592 prevBB->getExit()->asFlow()->target.bb = convBB;
3593 }
3594 setPosition(convBB, true);
3595 }
3596 break;
3597 case TGSI_OPCODE_BGNLOOP:
3598 {
3599 BasicBlock *lbgnBB = new BasicBlock(func);
3600 BasicBlock *lbrkBB = new BasicBlock(func);
3601
3602 loopBBs.push(lbgnBB);
3603 breakBBs.push(lbrkBB);
3604 if (loopBBs.getSize() > func->loopNestingBound)
3605 func->loopNestingBound++;
3606
3607 mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3608
3609 bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3610 setPosition(lbgnBB, true);
3611 mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3612 }
3613 break;
3614 case TGSI_OPCODE_ENDLOOP:
3615 {
3616 BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3617
3618 if (!bb->isTerminated()) {
3619 mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3620 bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3621 }
3622 setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3623
3624 // If the loop never breaks (e.g. only has RET's inside), then there
3625 // will be no way to get to the break bb. However BGNLOOP will have
3626 // already made a PREBREAK to it, so it must be in the CFG.
3627 if (getBB()->cfg.incidentCount() == 0)
3628 loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3629 }
3630 break;
3631 case TGSI_OPCODE_BRK:
3632 {
3633 if (bb->isTerminated())
3634 break;
3635 BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3636 mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3637 bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3638 }
3639 break;
3640 case TGSI_OPCODE_CONT:
3641 {
3642 if (bb->isTerminated())
3643 break;
3644 BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3645 mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3646 contBB->explicitCont = true;
3647 bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3648 }
3649 break;
3650 case TGSI_OPCODE_BGNSUB:
3651 {
3652 Subroutine *s = getSubroutine(ip);
3653 BasicBlock *entry = new BasicBlock(s->f);
3654 BasicBlock *leave = new BasicBlock(s->f);
3655
3656 // multiple entrypoints possible, keep the graph connected
3657 if (prog->getType() == Program::TYPE_COMPUTE)
3658 prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3659
3660 sub.cur = s;
3661 s->f->setEntry(entry);
3662 s->f->setExit(leave);
3663 setPosition(entry, true);
3664 return true;
3665 }
3666 case TGSI_OPCODE_ENDSUB:
3667 {
3668 sub.cur = getSubroutine(prog->main);
3669 setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3670 return true;
3671 }
3672 case TGSI_OPCODE_CAL:
3673 {
3674 Subroutine *s = getSubroutine(tgsi.getLabel());
3675 mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3676 func->call.attach(&s->f->call, Graph::Edge::TREE);
3677 return true;
3678 }
3679 case TGSI_OPCODE_RET:
3680 {
3681 if (bb->isTerminated())
3682 return true;
3683 BasicBlock *leave = BasicBlock::get(func->cfgExit);
3684
3685 if (!isEndOfSubroutine(ip + 1)) {
3686 // insert a PRERET at the entry if this is an early return
3687 // (only needed for sharing code in the epilogue)
3688 BasicBlock *root = BasicBlock::get(func->cfg.getRoot());
3689 if (root->getEntry() == NULL || root->getEntry()->op != OP_PRERET) {
3690 BasicBlock *pos = getBB();
3691 setPosition(root, false);
3692 mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3693 setPosition(pos, true);
3694 }
3695 }
3696 mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3697 bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3698 }
3699 break;
3700 case TGSI_OPCODE_END:
3701 {
3702 // attach and generate epilogue code
3703 BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3704 bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3705 setPosition(epilogue, true);
3706 if (prog->getType() == Program::TYPE_FRAGMENT)
3707 exportOutputs();
3708 if ((prog->getType() == Program::TYPE_VERTEX ||
3709 prog->getType() == Program::TYPE_TESSELLATION_EVAL
3710 ) && info_out->io.genUserClip > 0)
3711 handleUserClipPlanes();
3712 mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3713 }
3714 break;
3715 case TGSI_OPCODE_SWITCH:
3716 case TGSI_OPCODE_CASE:
3717 ERROR("switch/case opcode encountered, should have been lowered\n");
3718 abort();
3719 break;
3720 case TGSI_OPCODE_LOAD:
3721 handleLOAD(dst0);
3722 break;
3723 case TGSI_OPCODE_STORE:
3724 handleSTORE();
3725 break;
3726 case TGSI_OPCODE_BARRIER:
3727 geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3728 geni->fixed = 1;
3729 geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3730 break;
3731 case TGSI_OPCODE_MEMBAR:
3732 {
3733 uint32_t level = tgsi.getSrc(0).getValueU32(0, code->immd.data);
3734 geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3735 geni->fixed = 1;
3736 if (!(level & ~(TGSI_MEMBAR_THREAD_GROUP | TGSI_MEMBAR_SHARED)))
3737 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3738 else
3739 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3740 }
3741 break;
3742 case TGSI_OPCODE_ATOMUADD:
3743 case TGSI_OPCODE_ATOMXCHG:
3744 case TGSI_OPCODE_ATOMCAS:
3745 case TGSI_OPCODE_ATOMAND:
3746 case TGSI_OPCODE_ATOMOR:
3747 case TGSI_OPCODE_ATOMXOR:
3748 case TGSI_OPCODE_ATOMUMIN:
3749 case TGSI_OPCODE_ATOMIMIN:
3750 case TGSI_OPCODE_ATOMUMAX:
3751 case TGSI_OPCODE_ATOMIMAX:
3752 case TGSI_OPCODE_ATOMFADD:
3753 case TGSI_OPCODE_ATOMDEC_WRAP:
3754 case TGSI_OPCODE_ATOMINC_WRAP:
3755 handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3756 break;
3757 case TGSI_OPCODE_RESQ:
3758 if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER) {
3759 Value *ind = NULL;
3760 if (tgsi.getSrc(0).isIndirect(0))
3761 ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
3762 geni = mkOp1(OP_BUFQ, TYPE_U32, dst0[0],
3763 makeSym(tgsi.getSrc(0).getFile(),
3764 tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3765 if (ind)
3766 geni->setIndirect(0, 1, ind);
3767 } else {
3768 TexInstruction *texi = new_TexInstruction(func, OP_SUQ);
3769 for (int c = 0, d = 0; c < 4; ++c) {
3770 if (dst0[c]) {
3771 texi->setDef(d++, dst0[c]);
3772 texi->tex.mask |= 1 << c;
3773 }
3774 }
3775 if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMAGE) {
3776 texi->tex.r = tgsi.getSrc(0).getIndex(0);
3777 if (tgsi.getSrc(0).isIndirect(0))
3778 texi->setIndirectR(fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, NULL));
3779 } else {
3780 texi->tex.bindless = true;
3781 texi->setIndirectR(fetchSrc(0, 0));
3782 }
3783 texi->tex.target = tgsi.getImageTarget();
3784
3785 bb->insertTail(texi);
3786 }
3787 break;
3788 case TGSI_OPCODE_IBFE:
3789 case TGSI_OPCODE_UBFE:
3790 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3791 src0 = fetchSrc(0, c);
3792 val0 = getScratch();
3793 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3794 tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3795 loadImm(val0, (tgsi.getSrc(2).getValueU32(c, code->immd.data) << 8) |
3796 tgsi.getSrc(1).getValueU32(c, code->immd.data));
3797 } else {
3798 src1 = fetchSrc(1, c);
3799 src2 = fetchSrc(2, c);
3800 mkOp3(OP_INSBF, TYPE_U32, val0, src2, mkImm(0x808), src1);
3801 }
3802 mkOp2(OP_EXTBF, dstTy, dst0[c], src0, val0);
3803 }
3804 break;
3805 case TGSI_OPCODE_BFI:
3806 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3807 src0 = fetchSrc(0, c);
3808 src1 = fetchSrc(1, c);
3809 src2 = fetchSrc(2, c);
3810 src3 = fetchSrc(3, c);
3811 val0 = getScratch();
3812 mkOp3(OP_INSBF, TYPE_U32, val0, src3, mkImm(0x808), src2);
3813 mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, val0, src0);
3814 }
3815 break;
3816 case TGSI_OPCODE_LSB:
3817 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3818 src0 = fetchSrc(0, c);
3819 val0 = getScratch();
3820 mkOp1(OP_BREV, TYPE_U32, val0, src0);
3821 geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], val0);
3822 geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3823 }
3824 break;
3825 case TGSI_OPCODE_IMSB:
3826 case TGSI_OPCODE_UMSB:
3827 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3828 src0 = fetchSrc(0, c);
3829 mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3830 }
3831 break;
3832 case TGSI_OPCODE_BREV:
3833 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3834 src0 = fetchSrc(0, c);
3835 mkOp1(OP_BREV, TYPE_U32, dst0[c], src0);
3836 }
3837 break;
3838 case TGSI_OPCODE_POPC:
3839 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3840 src0 = fetchSrc(0, c);
3841 mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3842 }
3843 break;
3844 case TGSI_OPCODE_INTERP_CENTROID:
3845 case TGSI_OPCODE_INTERP_SAMPLE:
3846 case TGSI_OPCODE_INTERP_OFFSET:
3847 handleINTERP(dst0);
3848 break;
3849 case TGSI_OPCODE_I642F:
3850 case TGSI_OPCODE_U642F:
3851 case TGSI_OPCODE_D2I:
3852 case TGSI_OPCODE_D2U:
3853 case TGSI_OPCODE_D2F: {
3854 int pos = 0;
3855 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3856 Value *dreg = getSSA(8);
3857 src0 = fetchSrc(0, pos);
3858 src1 = fetchSrc(0, pos + 1);
3859 mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3860 Instruction *cvt = mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3861 if (!isFloatType(dstTy))
3862 cvt->rnd = ROUND_Z;
3863 pos += 2;
3864 }
3865 break;
3866 }
3867 case TGSI_OPCODE_I2I64:
3868 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3869 dst0[c] = fetchSrc(0, c / 2);
3870 mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(NULL, 31));
3871 c++;
3872 }
3873 break;
3874 case TGSI_OPCODE_U2I64:
3875 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3876 dst0[c] = fetchSrc(0, c / 2);
3877 dst0[c + 1] = zero;
3878 c++;
3879 }
3880 break;
3881 case TGSI_OPCODE_F2I64:
3882 case TGSI_OPCODE_F2U64:
3883 case TGSI_OPCODE_I2D:
3884 case TGSI_OPCODE_U2D:
3885 case TGSI_OPCODE_F2D:
3886 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3887 Value *dreg = getSSA(8);
3888 Instruction *cvt = mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3889 if (!isFloatType(dstTy))
3890 cvt->rnd = ROUND_Z;
3891 mkSplit(&dst0[c], 4, dreg);
3892 c++;
3893 }
3894 break;
3895 case TGSI_OPCODE_D2I64:
3896 case TGSI_OPCODE_D2U64:
3897 case TGSI_OPCODE_I642D:
3898 case TGSI_OPCODE_U642D:
3899 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3900 src0 = getSSA(8);
3901 Value *dst = getSSA(8), *tmp[2];
3902 tmp[0] = fetchSrc(0, c);
3903 tmp[1] = fetchSrc(0, c + 1);
3904 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3905 Instruction *cvt = mkCvt(OP_CVT, dstTy, dst, srcTy, src0);
3906 if (!isFloatType(dstTy))
3907 cvt->rnd = ROUND_Z;
3908 mkSplit(&dst0[c], 4, dst);
3909 c++;
3910 }
3911 break;
3912 case TGSI_OPCODE_I64NEG:
3913 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3914 src0 = getSSA(8);
3915 Value *dst = getSSA(8), *tmp[2];
3916 tmp[0] = fetchSrc(0, c);
3917 tmp[1] = fetchSrc(0, c + 1);
3918 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3919 mkOp2(OP_SUB, dstTy, dst, zero, src0);
3920 mkSplit(&dst0[c], 4, dst);
3921 c++;
3922 }
3923 break;
3924 case TGSI_OPCODE_I64ABS:
3925 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3926 src0 = getSSA(8);
3927 Value *neg = getSSA(8), *srcComp[2], *negComp[2];
3928 srcComp[0] = fetchSrc(0, c);
3929 srcComp[1] = fetchSrc(0, c + 1);
3930 mkOp2(OP_MERGE, TYPE_U64, src0, srcComp[0], srcComp[1]);
3931 mkOp2(OP_SUB, dstTy, neg, zero, src0);
3932 mkSplit(negComp, 4, neg);
3933 mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c], TYPE_S32,
3934 negComp[0], srcComp[0], srcComp[1]);
3935 mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c + 1], TYPE_S32,
3936 negComp[1], srcComp[1], srcComp[1]);
3937 c++;
3938 }
3939 break;
3940 case TGSI_OPCODE_DABS:
3941 case TGSI_OPCODE_DNEG:
3942 case TGSI_OPCODE_DRCP:
3943 case TGSI_OPCODE_DSQRT:
3944 case TGSI_OPCODE_DRSQ:
3945 case TGSI_OPCODE_DTRUNC:
3946 case TGSI_OPCODE_DCEIL:
3947 case TGSI_OPCODE_DFLR:
3948 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3949 src0 = getSSA(8);
3950 Value *dst = getSSA(8), *tmp[2];
3951 tmp[0] = fetchSrc(0, c);
3952 tmp[1] = fetchSrc(0, c + 1);
3953 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3954 mkOp1(op, dstTy, dst, src0);
3955 mkSplit(&dst0[c], 4, dst);
3956 c++;
3957 }
3958 break;
3959 case TGSI_OPCODE_DFRAC:
3960 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3961 src0 = getSSA(8);
3962 Value *dst = getSSA(8), *tmp[2];
3963 tmp[0] = fetchSrc(0, c);
3964 tmp[1] = fetchSrc(0, c + 1);
3965 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3966 mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3967 mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3968 mkSplit(&dst0[c], 4, dst);
3969 c++;
3970 }
3971 break;
3972 case TGSI_OPCODE_U64SEQ:
3973 case TGSI_OPCODE_U64SNE:
3974 case TGSI_OPCODE_U64SLT:
3975 case TGSI_OPCODE_U64SGE:
3976 case TGSI_OPCODE_I64SLT:
3977 case TGSI_OPCODE_I64SGE:
3978 case TGSI_OPCODE_DSLT:
3979 case TGSI_OPCODE_DSGE:
3980 case TGSI_OPCODE_DSEQ:
3981 case TGSI_OPCODE_DSNE: {
3982 int pos = 0;
3983 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3984 Value *tmp[2];
3985
3986 src0 = getSSA(8);
3987 src1 = getSSA(8);
3988 tmp[0] = fetchSrc(0, pos);
3989 tmp[1] = fetchSrc(0, pos + 1);
3990 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3991 tmp[0] = fetchSrc(1, pos);
3992 tmp[1] = fetchSrc(1, pos + 1);
3993 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3994 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3995 pos += 2;
3996 }
3997 break;
3998 }
3999 case TGSI_OPCODE_U64MIN:
4000 case TGSI_OPCODE_U64MAX:
4001 case TGSI_OPCODE_I64MIN:
4002 case TGSI_OPCODE_I64MAX: {
4003 dstTy = isSignedIntType(dstTy) ? TYPE_S32 : TYPE_U32;
4004 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4005 Value *flag = getSSA(1, FILE_FLAGS);
4006 src0 = fetchSrc(0, c + 1);
4007 src1 = fetchSrc(1, c + 1);
4008 geni = mkOp2(op, dstTy, dst0[c + 1], src0, src1);
4009 geni->subOp = NV50_IR_SUBOP_MINMAX_HIGH;
4010 geni->setFlagsDef(1, flag);
4011
4012 src0 = fetchSrc(0, c);
4013 src1 = fetchSrc(1, c);
4014 geni = mkOp2(op, TYPE_U32, dst0[c], src0, src1);
4015 geni->subOp = NV50_IR_SUBOP_MINMAX_LOW;
4016 geni->setFlagsSrc(2, flag);
4017
4018 c++;
4019 }
4020 break;
4021 }
4022 case TGSI_OPCODE_U64SHL:
4023 case TGSI_OPCODE_I64SHR:
4024 case TGSI_OPCODE_U64SHR:
4025 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4026 src0 = getSSA(8);
4027 Value *dst = getSSA(8), *tmp[2];
4028 tmp[0] = fetchSrc(0, c);
4029 tmp[1] = fetchSrc(0, c + 1);
4030 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4031 // Theoretically src1 is a 64-bit value but in practice only the low
4032 // bits matter. The IR expects this to be a 32-bit value.
4033 src1 = fetchSrc(1, c);
4034 mkOp2(op, dstTy, dst, src0, src1);
4035 mkSplit(&dst0[c], 4, dst);
4036 c++;
4037 }
4038 break;
4039 case TGSI_OPCODE_U64ADD:
4040 case TGSI_OPCODE_U64MUL:
4041 case TGSI_OPCODE_DADD:
4042 case TGSI_OPCODE_DMUL:
4043 case TGSI_OPCODE_DDIV:
4044 case TGSI_OPCODE_DMAX:
4045 case TGSI_OPCODE_DMIN:
4046 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4047 src0 = getSSA(8);
4048 src1 = getSSA(8);
4049 Value *dst = getSSA(8), *tmp[2];
4050 tmp[0] = fetchSrc(0, c);
4051 tmp[1] = fetchSrc(0, c + 1);
4052 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4053 tmp[0] = fetchSrc(1, c);
4054 tmp[1] = fetchSrc(1, c + 1);
4055 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4056 mkOp2(op, dstTy, dst, src0, src1);
4057 mkSplit(&dst0[c], 4, dst);
4058 c++;
4059 }
4060 break;
4061 case TGSI_OPCODE_DMAD:
4062 case TGSI_OPCODE_DFMA:
4063 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4064 src0 = getSSA(8);
4065 src1 = getSSA(8);
4066 src2 = getSSA(8);
4067 Value *dst = getSSA(8), *tmp[2];
4068 tmp[0] = fetchSrc(0, c);
4069 tmp[1] = fetchSrc(0, c + 1);
4070 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4071 tmp[0] = fetchSrc(1, c);
4072 tmp[1] = fetchSrc(1, c + 1);
4073 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4074 tmp[0] = fetchSrc(2, c);
4075 tmp[1] = fetchSrc(2, c + 1);
4076 mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
4077 mkOp3(op, dstTy, dst, src0, src1, src2);
4078 mkSplit(&dst0[c], 4, dst);
4079 c++;
4080 }
4081 break;
4082 case TGSI_OPCODE_DROUND:
4083 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4084 src0 = getSSA(8);
4085 Value *dst = getSSA(8), *tmp[2];
4086 tmp[0] = fetchSrc(0, c);
4087 tmp[1] = fetchSrc(0, c + 1);
4088 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4089 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
4090 ->rnd = ROUND_NI;
4091 mkSplit(&dst0[c], 4, dst);
4092 c++;
4093 }
4094 break;
4095 case TGSI_OPCODE_DSSG:
4096 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4097 src0 = getSSA(8);
4098 Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
4099 tmp[0] = fetchSrc(0, c);
4100 tmp[1] = fetchSrc(0, c + 1);
4101 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4102
4103 val0 = getScratch();
4104 val1 = getScratch();
4105 // The zero is wrong here since it's only 32-bit, but it works out in
4106 // the end since it gets replaced with $r63.
4107 mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
4108 mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
4109 mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
4110 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
4111 mkSplit(&dst0[c], 4, dst);
4112 c++;
4113 }
4114 break;
4115 case TGSI_OPCODE_I64SSG:
4116 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4117 src0 = getSSA(8);
4118 Value *tmp[2];
4119 tmp[0] = fetchSrc(0, c);
4120 tmp[1] = fetchSrc(0, c + 1);
4121 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4122
4123 val0 = getScratch();
4124 val1 = getScratch();
4125 mkCmp(OP_SET, CC_GT, TYPE_U32, val0, TYPE_S64, src0, zero);
4126 mkCmp(OP_SET, CC_LT, TYPE_U32, val1, TYPE_S64, src0, zero);
4127 mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
4128 mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(0, 31));
4129 c++;
4130 }
4131 break;
4132 default:
4133 ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
4134 assert(0);
4135 break;
4136 }
4137
4138 if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
4139 for (c = 0; c < 4; ++c) {
4140 if (!dst0[c])
4141 continue;
4142 if (dst0[c] != rDst0[c])
4143 mkMov(rDst0[c], dst0[c]);
4144 storeDst(0, c, rDst0[c]);
4145 }
4146 }
4147 vtxBaseValid = 0;
4148
4149 return true;
4150 }
4151
4152 void
exportOutputs()4153 Converter::exportOutputs()
4154 {
4155 if (info->io.alphaRefBase) {
4156 for (unsigned int i = 0; i < info_out->numOutputs; ++i) {
4157 if (info_out->out[i].sn != TGSI_SEMANTIC_COLOR ||
4158 info_out->out[i].si != 0)
4159 continue;
4160 const unsigned int c = 3;
4161 if (!oData.exists(sub.cur->values, i, c))
4162 continue;
4163 Value *val = oData.load(sub.cur->values, i, c, NULL);
4164 if (!val)
4165 continue;
4166
4167 Symbol *ref = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
4168 TYPE_U32, info->io.alphaRefBase);
4169 Value *pred = new_LValue(func, FILE_PREDICATE);
4170 mkCmp(OP_SET, CC_TR, TYPE_U32, pred, TYPE_F32, val,
4171 mkLoadv(TYPE_U32, ref, NULL))
4172 ->subOp = 1;
4173 mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_NOT_P, pred);
4174 }
4175 }
4176
4177 for (unsigned int i = 0; i < info_out->numOutputs; ++i) {
4178 for (unsigned int c = 0; c < 4; ++c) {
4179 if (!oData.exists(sub.cur->values, i, c))
4180 continue;
4181 Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
4182 info_out->out[i].slot[c] * 4);
4183 Value *val = oData.load(sub.cur->values, i, c, NULL);
4184 if (val) {
4185 if (info_out->out[i].sn == TGSI_SEMANTIC_POSITION)
4186 mkOp1(OP_SAT, TYPE_F32, val, val);
4187 mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
4188 }
4189 }
4190 }
4191 }
4192
Converter(Program * ir,const tgsi::Source * code,nv50_ir_prog_info_out * info_out)4193 Converter::Converter(Program *ir, const tgsi::Source *code, nv50_ir_prog_info_out *info_out)
4194 : ConverterCommon(ir, code->info, info_out),
4195 code(code),
4196 tgsi(NULL),
4197 tData(this), lData(this), aData(this), oData(this)
4198 {
4199 const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
4200 const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
4201 const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
4202
4203 tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
4204 lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
4205 aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
4206 oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
4207
4208 zero = mkImm((uint32_t)0);
4209
4210 vtxBaseValid = 0;
4211 }
4212
~Converter()4213 Converter::~Converter()
4214 {
4215 }
4216
4217 inline const Converter::Location *
getValueLocation(Subroutine * s,Value * v)4218 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
4219 {
4220 ValueMap::l_iterator it = s->values.l.find(v);
4221 return it == s->values.l.end() ? NULL : &it->second;
4222 }
4223
4224 template<typename T> inline void
updateCallArgs(Instruction * i,void (Instruction::* setArg)(int,Value *),T (Function::* proto))4225 Converter::BindArgumentsPass::updateCallArgs(
4226 Instruction *i, void (Instruction::*setArg)(int, Value *),
4227 T (Function::*proto))
4228 {
4229 Function *g = i->asFlow()->target.fn;
4230 Subroutine *subg = conv.getSubroutine(g);
4231
4232 for (unsigned a = 0; a < (g->*proto).size(); ++a) {
4233 Value *v = (g->*proto)[a].get();
4234 const Converter::Location &l = *getValueLocation(subg, v);
4235 Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
4236
4237 (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
4238 }
4239 }
4240
4241 template<typename T> inline void
updatePrototype(BitSet * set,void (Function::* updateSet)(),T (Function::* proto))4242 Converter::BindArgumentsPass::updatePrototype(
4243 BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
4244 {
4245 (func->*updateSet)();
4246
4247 for (unsigned i = 0; i < set->getSize(); ++i) {
4248 Value *v = func->getLValue(i);
4249 const Converter::Location *l = getValueLocation(sub, v);
4250
4251 // only include values with a matching TGSI register
4252 if (set->test(i) && l && !conv.code->locals.count(*l))
4253 (func->*proto).push_back(v);
4254 }
4255 }
4256
4257 bool
visit(Function * f)4258 Converter::BindArgumentsPass::visit(Function *f)
4259 {
4260 sub = conv.getSubroutine(f);
4261
4262 for (ArrayList::Iterator bi = f->allBBlocks.iterator();
4263 !bi.end(); bi.next()) {
4264 for (Instruction *i = BasicBlock::get(bi)->getFirst();
4265 i; i = i->next) {
4266 if (i->op == OP_CALL && !i->asFlow()->builtin) {
4267 updateCallArgs(i, &Instruction::setSrc, &Function::ins);
4268 updateCallArgs(i, &Instruction::setDef, &Function::outs);
4269 }
4270 }
4271 }
4272
4273 if (func == prog->main /* && prog->getType() != Program::TYPE_COMPUTE */)
4274 return true;
4275 updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
4276 &Function::buildLiveSets, &Function::ins);
4277 updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
4278 &Function::buildDefSets, &Function::outs);
4279
4280 return true;
4281 }
4282
4283 bool
run()4284 Converter::run()
4285 {
4286 BasicBlock *entry = new BasicBlock(prog->main);
4287 BasicBlock *leave = new BasicBlock(prog->main);
4288
4289 prog->main->setEntry(entry);
4290 prog->main->setExit(leave);
4291
4292 setPosition(entry, true);
4293 sub.cur = getSubroutine(prog->main);
4294
4295 if (info_out->io.genUserClip > 0) {
4296 for (int c = 0; c < 4; ++c)
4297 clipVtx[c] = getScratch();
4298 }
4299
4300 switch (prog->getType()) {
4301 case Program::TYPE_TESSELLATION_CONTROL:
4302 outBase = mkOp2v(
4303 OP_SUB, TYPE_U32, getSSA(),
4304 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
4305 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
4306 break;
4307 case Program::TYPE_FRAGMENT: {
4308 Symbol *sv = mkSysVal(SV_POSITION, 3);
4309 fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
4310 mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
4311 break;
4312 }
4313 default:
4314 break;
4315 }
4316
4317 if (info->io.viewportId >= 0)
4318 viewport = getScratch();
4319 else
4320 viewport = NULL;
4321
4322 for (ip = 0; ip < code->scan.num_instructions; ++ip) {
4323 if (!handleInstruction(&code->insns[ip]))
4324 return false;
4325 }
4326
4327 if (!BindArgumentsPass(*this).run(prog))
4328 return false;
4329
4330 return true;
4331 }
4332
4333 } // unnamed namespace
4334
4335 namespace nv50_ir {
4336
4337 bool
makeFromTGSI(struct nv50_ir_prog_info * info,struct nv50_ir_prog_info_out * info_out)4338 Program::makeFromTGSI(struct nv50_ir_prog_info *info,
4339 struct nv50_ir_prog_info_out *info_out)
4340 {
4341 tgsi::Source src(info, info_out, this);
4342 if (!src.scanSource())
4343 return false;
4344 tlsSize = info_out->bin.tlsSpace;
4345
4346 Converter builder(this, &src, info_out);
4347 return builder.run();
4348 }
4349
4350 } // namespace nv50_ir
4351