• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2011 Christoph Bumiller
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include "tgsi/tgsi_build.h"
24 #include "tgsi/tgsi_dump.h"
25 #include "tgsi/tgsi_scan.h"
26 #include "tgsi/tgsi_util.h"
27 
28 #include <set>
29 
30 #include "codegen/nv50_ir.h"
31 #include "codegen/nv50_ir_from_common.h"
32 #include "codegen/nv50_ir_util.h"
33 
34 namespace tgsi {
35 
36 class Source;
37 
38 static nv50_ir::operation translateOpcode(uint opcode);
39 static nv50_ir::DataFile translateFile(uint file);
40 static nv50_ir::TexTarget translateTexture(uint texTarg);
41 static nv50_ir::SVSemantic translateSysVal(uint sysval);
42 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
43 
44 class Instruction
45 {
46 public:
Instruction(const struct tgsi_full_instruction * inst)47    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
48 
49    class SrcRegister
50    {
51    public:
SrcRegister(const struct tgsi_full_src_register * src)52       SrcRegister(const struct tgsi_full_src_register *src)
53          : reg(src->Register),
54            fsr(src)
55       { }
56 
SrcRegister(const struct tgsi_src_register & src)57       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
58 
SrcRegister(const struct tgsi_ind_register & ind)59       SrcRegister(const struct tgsi_ind_register& ind)
60          : reg(tgsi_util_get_src_from_ind(&ind)),
61            fsr(NULL)
62       { }
63 
offsetToSrc(struct tgsi_texture_offset off)64       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
65       {
66          struct tgsi_src_register reg;
67          memset(&reg, 0, sizeof(reg));
68          reg.Index = off.Index;
69          reg.File = off.File;
70          reg.SwizzleX = off.SwizzleX;
71          reg.SwizzleY = off.SwizzleY;
72          reg.SwizzleZ = off.SwizzleZ;
73          return reg;
74       }
75 
SrcRegister(const struct tgsi_texture_offset & off)76       SrcRegister(const struct tgsi_texture_offset& off) :
77          reg(offsetToSrc(off)),
78          fsr(NULL)
79       { }
80 
getFile() const81       uint getFile() const { return reg.File; }
82 
is2D() const83       bool is2D() const { return reg.Dimension; }
84 
isIndirect(int dim) const85       bool isIndirect(int dim) const
86       {
87          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
88       }
89 
getIndex(int dim) const90       int getIndex(int dim) const
91       {
92          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
93       }
94 
getSwizzle(int chan) const95       int getSwizzle(int chan) const
96       {
97          return tgsi_util_get_src_register_swizzle(&reg, chan);
98       }
99 
getArrayId() const100       int getArrayId() const
101       {
102          if (isIndirect(0))
103             return fsr->Indirect.ArrayID;
104          return 0;
105       }
106 
107       nv50_ir::Modifier getMod(int chan) const;
108 
getIndirect(int dim) const109       SrcRegister getIndirect(int dim) const
110       {
111          assert(fsr && isIndirect(dim));
112          if (dim)
113             return SrcRegister(fsr->DimIndirect);
114          return SrcRegister(fsr->Indirect);
115       }
116 
getValueU32(int c,const uint32_t * data) const117       uint32_t getValueU32(int c, const uint32_t *data) const
118       {
119          assert(reg.File == TGSI_FILE_IMMEDIATE);
120          assert(!reg.Absolute);
121          assert(!reg.Negate);
122          return data[reg.Index * 4 + getSwizzle(c)];
123       }
124 
125    private:
126       const struct tgsi_src_register reg;
127       const struct tgsi_full_src_register *fsr;
128    };
129 
130    class DstRegister
131    {
132    public:
DstRegister(const struct tgsi_full_dst_register * dst)133       DstRegister(const struct tgsi_full_dst_register *dst)
134          : reg(dst->Register),
135            fdr(dst)
136       { }
137 
DstRegister(const struct tgsi_dst_register & dst)138       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
139 
getFile() const140       uint getFile() const { return reg.File; }
141 
is2D() const142       bool is2D() const { return reg.Dimension; }
143 
isIndirect(int dim) const144       bool isIndirect(int dim) const
145       {
146          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
147       }
148 
getIndex(int dim) const149       int getIndex(int dim) const
150       {
151          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
152       }
153 
getMask() const154       unsigned int getMask() const { return reg.WriteMask; }
155 
isMasked(int chan) const156       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
157 
getIndirect(int dim) const158       SrcRegister getIndirect(int dim) const
159       {
160          assert(fdr && isIndirect(dim));
161          if (dim)
162             return SrcRegister(fdr->DimIndirect);
163          return SrcRegister(fdr->Indirect);
164       }
165 
asSrc()166       struct tgsi_full_src_register asSrc()
167       {
168          assert(fdr);
169          return tgsi_full_src_register_from_dst(fdr);
170       }
171 
getArrayId() const172       int getArrayId() const
173       {
174          if (isIndirect(0))
175             return fdr->Indirect.ArrayID;
176          return 0;
177       }
178 
179    private:
180       const struct tgsi_dst_register reg;
181       const struct tgsi_full_dst_register *fdr;
182    };
183 
getOpcode() const184    inline uint getOpcode() const { return insn->Instruction.Opcode; }
185 
srcCount() const186    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
dstCount() const187    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
188 
189    // mask of used components of source s
190    unsigned int srcMask(unsigned int s) const;
191    unsigned int texOffsetMask() const;
192 
getSrc(unsigned int s) const193    SrcRegister getSrc(unsigned int s) const
194    {
195       assert(s < srcCount());
196       return SrcRegister(&insn->Src[s]);
197    }
198 
getDst(unsigned int d) const199    DstRegister getDst(unsigned int d) const
200    {
201       assert(d < dstCount());
202       return DstRegister(&insn->Dst[d]);
203    }
204 
getTexOffset(unsigned int i) const205    SrcRegister getTexOffset(unsigned int i) const
206    {
207       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
208       return SrcRegister(insn->TexOffsets[i]);
209    }
210 
getNumTexOffsets() const211    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
212 
213    bool checkDstSrcAliasing() const;
214 
getOP() const215    inline nv50_ir::operation getOP() const {
216       return translateOpcode(getOpcode()); }
217 
218    nv50_ir::DataType inferSrcType() const;
219    nv50_ir::DataType inferDstType() const;
220 
221    nv50_ir::CondCode getSetCond() const;
222 
223    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
224 
getImageFormat() const225    const nv50_ir::TexInstruction::ImgFormatDesc *getImageFormat() const {
226       return nv50_ir::TexInstruction::translateImgFormat((enum pipe_format)insn->Memory.Format);
227    }
228 
getImageTarget() const229    nv50_ir::TexTarget getImageTarget() const {
230       return translateTexture(insn->Memory.Texture);
231    }
232 
getCacheMode() const233    nv50_ir::CacheMode getCacheMode() const {
234       if (!insn->Instruction.Memory)
235          return nv50_ir::CACHE_CA;
236       return translateCacheMode(insn->Memory.Qualifier);
237    }
238 
getLabel()239    inline uint getLabel() { return insn->Label.Label; }
240 
getSaturate() const241    unsigned getSaturate() const { return insn->Instruction.Saturate; }
242 
print() const243    void print() const
244    {
245       tgsi_dump_instruction(insn, 1);
246    }
247 
248 private:
249    const struct tgsi_full_instruction *insn;
250 };
251 
texOffsetMask() const252 unsigned int Instruction::texOffsetMask() const
253 {
254    const struct tgsi_instruction_texture *tex = &insn->Texture;
255    assert(insn->Instruction.Texture);
256 
257    switch (tex->Texture) {
258    case TGSI_TEXTURE_BUFFER:
259    case TGSI_TEXTURE_1D:
260    case TGSI_TEXTURE_SHADOW1D:
261    case TGSI_TEXTURE_1D_ARRAY:
262    case TGSI_TEXTURE_SHADOW1D_ARRAY:
263       return 0x1;
264    case TGSI_TEXTURE_2D:
265    case TGSI_TEXTURE_SHADOW2D:
266    case TGSI_TEXTURE_2D_ARRAY:
267    case TGSI_TEXTURE_SHADOW2D_ARRAY:
268    case TGSI_TEXTURE_RECT:
269    case TGSI_TEXTURE_SHADOWRECT:
270    case TGSI_TEXTURE_2D_MSAA:
271    case TGSI_TEXTURE_2D_ARRAY_MSAA:
272       return 0x3;
273    case TGSI_TEXTURE_3D:
274       return 0x7;
275    default:
276       assert(!"Unexpected texture target");
277       return 0xf;
278    }
279 }
280 
srcMask(unsigned int s) const281 unsigned int Instruction::srcMask(unsigned int s) const
282 {
283    unsigned int mask = insn->Dst[0].Register.WriteMask;
284 
285    switch (insn->Instruction.Opcode) {
286    case TGSI_OPCODE_COS:
287    case TGSI_OPCODE_SIN:
288       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
289    case TGSI_OPCODE_DP2:
290       return 0x3;
291    case TGSI_OPCODE_DP3:
292       return 0x7;
293    case TGSI_OPCODE_DP4:
294    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
295       return 0xf;
296    case TGSI_OPCODE_DST:
297       return mask & (s ? 0xa : 0x6);
298    case TGSI_OPCODE_EX2:
299    case TGSI_OPCODE_EXP:
300    case TGSI_OPCODE_LG2:
301    case TGSI_OPCODE_LOG:
302    case TGSI_OPCODE_POW:
303    case TGSI_OPCODE_RCP:
304    case TGSI_OPCODE_RSQ:
305       return 0x1;
306    case TGSI_OPCODE_IF:
307    case TGSI_OPCODE_UIF:
308       return 0x1;
309    case TGSI_OPCODE_LIT:
310       return 0xb;
311    case TGSI_OPCODE_TEX2:
312    case TGSI_OPCODE_TXB2:
313    case TGSI_OPCODE_TXL2:
314       return (s == 0) ? 0xf : 0x3;
315    case TGSI_OPCODE_TEX:
316    case TGSI_OPCODE_TXB:
317    case TGSI_OPCODE_TXD:
318    case TGSI_OPCODE_TXL:
319    case TGSI_OPCODE_TXP:
320    case TGSI_OPCODE_TXF:
321    case TGSI_OPCODE_TG4:
322    case TGSI_OPCODE_TEX_LZ:
323    case TGSI_OPCODE_TXF_LZ:
324    case TGSI_OPCODE_LODQ:
325    {
326       const struct tgsi_instruction_texture *tex = &insn->Texture;
327 
328       assert(insn->Instruction.Texture);
329 
330       mask = 0x7;
331       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
332           insn->Instruction.Opcode != TGSI_OPCODE_TEX_LZ &&
333           insn->Instruction.Opcode != TGSI_OPCODE_TXF_LZ &&
334           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
335          mask |= 0x8; /* bias, lod or proj */
336 
337       switch (tex->Texture) {
338       case TGSI_TEXTURE_1D:
339          mask &= 0x9;
340          break;
341       case TGSI_TEXTURE_SHADOW1D:
342          mask &= 0xd;
343          break;
344       case TGSI_TEXTURE_1D_ARRAY:
345       case TGSI_TEXTURE_2D:
346       case TGSI_TEXTURE_RECT:
347          mask &= 0xb;
348          break;
349       case TGSI_TEXTURE_CUBE_ARRAY:
350       case TGSI_TEXTURE_SHADOW2D_ARRAY:
351       case TGSI_TEXTURE_SHADOWCUBE:
352       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
353          mask |= 0x8;
354          break;
355       default:
356          break;
357       }
358    }
359       return mask;
360    case TGSI_OPCODE_TXQ:
361       return 1;
362    case TGSI_OPCODE_D2I:
363    case TGSI_OPCODE_D2U:
364    case TGSI_OPCODE_D2F:
365    case TGSI_OPCODE_DSLT:
366    case TGSI_OPCODE_DSGE:
367    case TGSI_OPCODE_DSEQ:
368    case TGSI_OPCODE_DSNE:
369    case TGSI_OPCODE_U64SEQ:
370    case TGSI_OPCODE_U64SNE:
371    case TGSI_OPCODE_I64SLT:
372    case TGSI_OPCODE_U64SLT:
373    case TGSI_OPCODE_I64SGE:
374    case TGSI_OPCODE_U64SGE:
375    case TGSI_OPCODE_I642F:
376    case TGSI_OPCODE_U642F:
377       switch (util_bitcount(mask)) {
378       case 1: return 0x3;
379       case 2: return 0xf;
380       default:
381          assert(!"unexpected mask");
382          return 0xf;
383       }
384    case TGSI_OPCODE_I2D:
385    case TGSI_OPCODE_U2D:
386    case TGSI_OPCODE_F2D: {
387       unsigned int x = 0;
388       if ((mask & 0x3) == 0x3)
389          x |= 1;
390       if ((mask & 0xc) == 0xc)
391          x |= 2;
392       return x;
393    }
394    case TGSI_OPCODE_PK2H:
395       return 0x3;
396    case TGSI_OPCODE_UP2H:
397       return 0x1;
398    default:
399       break;
400    }
401 
402    return mask;
403 }
404 
getMod(int chan) const405 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
406 {
407    nv50_ir::Modifier m(0);
408 
409    if (reg.Absolute)
410       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
411    if (reg.Negate)
412       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
413    return m;
414 }
415 
translateFile(uint file)416 static nv50_ir::DataFile translateFile(uint file)
417 {
418    switch (file) {
419    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
420    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
421    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
422    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
423    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
424    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
425    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
426    case TGSI_FILE_BUFFER:          return nv50_ir::FILE_MEMORY_BUFFER;
427    case TGSI_FILE_IMAGE:           return nv50_ir::FILE_MEMORY_GLOBAL;
428    case TGSI_FILE_MEMORY:          return nv50_ir::FILE_MEMORY_GLOBAL;
429    case TGSI_FILE_SAMPLER:
430    case TGSI_FILE_NULL:
431    default:
432       return nv50_ir::FILE_NULL;
433    }
434 }
435 
translateSysVal(uint sysval)436 static nv50_ir::SVSemantic translateSysVal(uint sysval)
437 {
438    switch (sysval) {
439    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
440    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
441    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
442    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
443    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
444    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
445    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
446    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
447    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
448    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
449    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
450    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
451    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
452    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
453    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
454    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
455    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
456    case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
457    case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
458    case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
459    case TGSI_SEMANTIC_DRAWID:     return nv50_ir::SV_DRAWID;
460    case TGSI_SEMANTIC_WORK_DIM:   return nv50_ir::SV_WORK_DIM;
461    case TGSI_SEMANTIC_SUBGROUP_INVOCATION: return nv50_ir::SV_LANEID;
462    case TGSI_SEMANTIC_SUBGROUP_EQ_MASK: return nv50_ir::SV_LANEMASK_EQ;
463    case TGSI_SEMANTIC_SUBGROUP_LT_MASK: return nv50_ir::SV_LANEMASK_LT;
464    case TGSI_SEMANTIC_SUBGROUP_LE_MASK: return nv50_ir::SV_LANEMASK_LE;
465    case TGSI_SEMANTIC_SUBGROUP_GT_MASK: return nv50_ir::SV_LANEMASK_GT;
466    case TGSI_SEMANTIC_SUBGROUP_GE_MASK: return nv50_ir::SV_LANEMASK_GE;
467    default:
468       assert(0);
469       return nv50_ir::SV_CLOCK;
470    }
471 }
472 
473 #define NV50_IR_TEX_TARG_CASE(a, b) \
474    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
475 
translateTexture(uint tex)476 static nv50_ir::TexTarget translateTexture(uint tex)
477 {
478    switch (tex) {
479    NV50_IR_TEX_TARG_CASE(1D, 1D);
480    NV50_IR_TEX_TARG_CASE(2D, 2D);
481    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
482    NV50_IR_TEX_TARG_CASE(3D, 3D);
483    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
484    NV50_IR_TEX_TARG_CASE(RECT, RECT);
485    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
486    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
487    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
488    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
489    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
490    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
491    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
492    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
493    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
494    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
495    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
496    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
497 
498    case TGSI_TEXTURE_UNKNOWN:
499    default:
500       assert(!"invalid texture target");
501       return nv50_ir::TEX_TARGET_2D;
502    }
503 }
504 
translateCacheMode(uint qualifier)505 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
506 {
507    if (qualifier & TGSI_MEMORY_VOLATILE)
508       return nv50_ir::CACHE_CV;
509    if (qualifier & TGSI_MEMORY_COHERENT)
510       return nv50_ir::CACHE_CG;
511    return nv50_ir::CACHE_CA;
512 }
513 
inferSrcType() const514 nv50_ir::DataType Instruction::inferSrcType() const
515 {
516    switch (getOpcode()) {
517    case TGSI_OPCODE_UIF:
518    case TGSI_OPCODE_AND:
519    case TGSI_OPCODE_OR:
520    case TGSI_OPCODE_XOR:
521    case TGSI_OPCODE_NOT:
522    case TGSI_OPCODE_SHL:
523    case TGSI_OPCODE_U2F:
524    case TGSI_OPCODE_U2D:
525    case TGSI_OPCODE_U2I64:
526    case TGSI_OPCODE_UADD:
527    case TGSI_OPCODE_UDIV:
528    case TGSI_OPCODE_UMOD:
529    case TGSI_OPCODE_UMAD:
530    case TGSI_OPCODE_UMUL:
531    case TGSI_OPCODE_UMUL_HI:
532    case TGSI_OPCODE_UMAX:
533    case TGSI_OPCODE_UMIN:
534    case TGSI_OPCODE_USEQ:
535    case TGSI_OPCODE_USGE:
536    case TGSI_OPCODE_USLT:
537    case TGSI_OPCODE_USNE:
538    case TGSI_OPCODE_USHR:
539    case TGSI_OPCODE_ATOMUADD:
540    case TGSI_OPCODE_ATOMXCHG:
541    case TGSI_OPCODE_ATOMCAS:
542    case TGSI_OPCODE_ATOMAND:
543    case TGSI_OPCODE_ATOMOR:
544    case TGSI_OPCODE_ATOMXOR:
545    case TGSI_OPCODE_ATOMUMIN:
546    case TGSI_OPCODE_ATOMUMAX:
547    case TGSI_OPCODE_ATOMDEC_WRAP:
548    case TGSI_OPCODE_ATOMINC_WRAP:
549    case TGSI_OPCODE_UBFE:
550    case TGSI_OPCODE_UMSB:
551    case TGSI_OPCODE_UP2H:
552    case TGSI_OPCODE_VOTE_ALL:
553    case TGSI_OPCODE_VOTE_ANY:
554    case TGSI_OPCODE_VOTE_EQ:
555       return nv50_ir::TYPE_U32;
556    case TGSI_OPCODE_I2F:
557    case TGSI_OPCODE_I2D:
558    case TGSI_OPCODE_I2I64:
559    case TGSI_OPCODE_IDIV:
560    case TGSI_OPCODE_IMUL_HI:
561    case TGSI_OPCODE_IMAX:
562    case TGSI_OPCODE_IMIN:
563    case TGSI_OPCODE_IABS:
564    case TGSI_OPCODE_INEG:
565    case TGSI_OPCODE_ISGE:
566    case TGSI_OPCODE_ISHR:
567    case TGSI_OPCODE_ISLT:
568    case TGSI_OPCODE_ISSG:
569    case TGSI_OPCODE_MOD:
570    case TGSI_OPCODE_UARL:
571    case TGSI_OPCODE_ATOMIMIN:
572    case TGSI_OPCODE_ATOMIMAX:
573    case TGSI_OPCODE_IBFE:
574    case TGSI_OPCODE_IMSB:
575       return nv50_ir::TYPE_S32;
576    case TGSI_OPCODE_D2F:
577    case TGSI_OPCODE_D2I:
578    case TGSI_OPCODE_D2U:
579    case TGSI_OPCODE_D2I64:
580    case TGSI_OPCODE_D2U64:
581    case TGSI_OPCODE_DABS:
582    case TGSI_OPCODE_DNEG:
583    case TGSI_OPCODE_DADD:
584    case TGSI_OPCODE_DMUL:
585    case TGSI_OPCODE_DDIV:
586    case TGSI_OPCODE_DMAX:
587    case TGSI_OPCODE_DMIN:
588    case TGSI_OPCODE_DSLT:
589    case TGSI_OPCODE_DSGE:
590    case TGSI_OPCODE_DSEQ:
591    case TGSI_OPCODE_DSNE:
592    case TGSI_OPCODE_DRCP:
593    case TGSI_OPCODE_DSQRT:
594    case TGSI_OPCODE_DMAD:
595    case TGSI_OPCODE_DFMA:
596    case TGSI_OPCODE_DFRAC:
597    case TGSI_OPCODE_DRSQ:
598    case TGSI_OPCODE_DTRUNC:
599    case TGSI_OPCODE_DCEIL:
600    case TGSI_OPCODE_DFLR:
601    case TGSI_OPCODE_DROUND:
602       return nv50_ir::TYPE_F64;
603    case TGSI_OPCODE_U64SEQ:
604    case TGSI_OPCODE_U64SNE:
605    case TGSI_OPCODE_U64SLT:
606    case TGSI_OPCODE_U64SGE:
607    case TGSI_OPCODE_U64MIN:
608    case TGSI_OPCODE_U64MAX:
609    case TGSI_OPCODE_U64ADD:
610    case TGSI_OPCODE_U64MUL:
611    case TGSI_OPCODE_U64SHL:
612    case TGSI_OPCODE_U64SHR:
613    case TGSI_OPCODE_U64DIV:
614    case TGSI_OPCODE_U64MOD:
615    case TGSI_OPCODE_U642F:
616    case TGSI_OPCODE_U642D:
617       return nv50_ir::TYPE_U64;
618    case TGSI_OPCODE_I64ABS:
619    case TGSI_OPCODE_I64SSG:
620    case TGSI_OPCODE_I64NEG:
621    case TGSI_OPCODE_I64SLT:
622    case TGSI_OPCODE_I64SGE:
623    case TGSI_OPCODE_I64MIN:
624    case TGSI_OPCODE_I64MAX:
625    case TGSI_OPCODE_I64SHR:
626    case TGSI_OPCODE_I64DIV:
627    case TGSI_OPCODE_I64MOD:
628    case TGSI_OPCODE_I642F:
629    case TGSI_OPCODE_I642D:
630       return nv50_ir::TYPE_S64;
631    default:
632       return nv50_ir::TYPE_F32;
633    }
634 }
635 
inferDstType() const636 nv50_ir::DataType Instruction::inferDstType() const
637 {
638    switch (getOpcode()) {
639    case TGSI_OPCODE_D2U:
640    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
641    case TGSI_OPCODE_D2I:
642    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
643    case TGSI_OPCODE_FSEQ:
644    case TGSI_OPCODE_FSGE:
645    case TGSI_OPCODE_FSLT:
646    case TGSI_OPCODE_FSNE:
647    case TGSI_OPCODE_DSEQ:
648    case TGSI_OPCODE_DSGE:
649    case TGSI_OPCODE_DSLT:
650    case TGSI_OPCODE_DSNE:
651    case TGSI_OPCODE_I64SLT:
652    case TGSI_OPCODE_I64SGE:
653    case TGSI_OPCODE_U64SEQ:
654    case TGSI_OPCODE_U64SNE:
655    case TGSI_OPCODE_U64SLT:
656    case TGSI_OPCODE_U64SGE:
657    case TGSI_OPCODE_PK2H:
658       return nv50_ir::TYPE_U32;
659    case TGSI_OPCODE_I2F:
660    case TGSI_OPCODE_U2F:
661    case TGSI_OPCODE_D2F:
662    case TGSI_OPCODE_I642F:
663    case TGSI_OPCODE_U642F:
664    case TGSI_OPCODE_UP2H:
665       return nv50_ir::TYPE_F32;
666    case TGSI_OPCODE_I2D:
667    case TGSI_OPCODE_U2D:
668    case TGSI_OPCODE_F2D:
669    case TGSI_OPCODE_I642D:
670    case TGSI_OPCODE_U642D:
671       return nv50_ir::TYPE_F64;
672    case TGSI_OPCODE_I2I64:
673    case TGSI_OPCODE_U2I64:
674    case TGSI_OPCODE_F2I64:
675    case TGSI_OPCODE_D2I64:
676       return nv50_ir::TYPE_S64;
677    case TGSI_OPCODE_F2U64:
678    case TGSI_OPCODE_D2U64:
679       return nv50_ir::TYPE_U64;
680    default:
681       return inferSrcType();
682    }
683 }
684 
getSetCond() const685 nv50_ir::CondCode Instruction::getSetCond() const
686 {
687    using namespace nv50_ir;
688 
689    switch (getOpcode()) {
690    case TGSI_OPCODE_SLT:
691    case TGSI_OPCODE_ISLT:
692    case TGSI_OPCODE_USLT:
693    case TGSI_OPCODE_FSLT:
694    case TGSI_OPCODE_DSLT:
695    case TGSI_OPCODE_I64SLT:
696    case TGSI_OPCODE_U64SLT:
697       return CC_LT;
698    case TGSI_OPCODE_SLE:
699       return CC_LE;
700    case TGSI_OPCODE_SGE:
701    case TGSI_OPCODE_ISGE:
702    case TGSI_OPCODE_USGE:
703    case TGSI_OPCODE_FSGE:
704    case TGSI_OPCODE_DSGE:
705    case TGSI_OPCODE_I64SGE:
706    case TGSI_OPCODE_U64SGE:
707       return CC_GE;
708    case TGSI_OPCODE_SGT:
709       return CC_GT;
710    case TGSI_OPCODE_SEQ:
711    case TGSI_OPCODE_USEQ:
712    case TGSI_OPCODE_FSEQ:
713    case TGSI_OPCODE_DSEQ:
714    case TGSI_OPCODE_U64SEQ:
715       return CC_EQ;
716    case TGSI_OPCODE_SNE:
717    case TGSI_OPCODE_FSNE:
718    case TGSI_OPCODE_DSNE:
719    case TGSI_OPCODE_U64SNE:
720       return CC_NEU;
721    case TGSI_OPCODE_USNE:
722       return CC_NE;
723    default:
724       return CC_ALWAYS;
725    }
726 }
727 
728 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
729 
translateOpcode(uint opcode)730 static nv50_ir::operation translateOpcode(uint opcode)
731 {
732    switch (opcode) {
733    NV50_IR_OPCODE_CASE(ARL, SHL);
734    NV50_IR_OPCODE_CASE(MOV, MOV);
735 
736    NV50_IR_OPCODE_CASE(RCP, RCP);
737    NV50_IR_OPCODE_CASE(RSQ, RSQ);
738    NV50_IR_OPCODE_CASE(SQRT, SQRT);
739 
740    NV50_IR_OPCODE_CASE(MUL, MUL);
741    NV50_IR_OPCODE_CASE(ADD, ADD);
742 
743    NV50_IR_OPCODE_CASE(MIN, MIN);
744    NV50_IR_OPCODE_CASE(MAX, MAX);
745    NV50_IR_OPCODE_CASE(SLT, SET);
746    NV50_IR_OPCODE_CASE(SGE, SET);
747    NV50_IR_OPCODE_CASE(MAD, MAD);
748    NV50_IR_OPCODE_CASE(FMA, FMA);
749 
750    NV50_IR_OPCODE_CASE(FLR, FLOOR);
751    NV50_IR_OPCODE_CASE(ROUND, CVT);
752    NV50_IR_OPCODE_CASE(EX2, EX2);
753    NV50_IR_OPCODE_CASE(LG2, LG2);
754    NV50_IR_OPCODE_CASE(POW, POW);
755 
756    NV50_IR_OPCODE_CASE(COS, COS);
757    NV50_IR_OPCODE_CASE(DDX, DFDX);
758    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
759    NV50_IR_OPCODE_CASE(DDY, DFDY);
760    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
761    NV50_IR_OPCODE_CASE(KILL, DISCARD);
762    NV50_IR_OPCODE_CASE(DEMOTE, DISCARD);
763 
764    NV50_IR_OPCODE_CASE(SEQ, SET);
765    NV50_IR_OPCODE_CASE(SGT, SET);
766    NV50_IR_OPCODE_CASE(SIN, SIN);
767    NV50_IR_OPCODE_CASE(SLE, SET);
768    NV50_IR_OPCODE_CASE(SNE, SET);
769    NV50_IR_OPCODE_CASE(TEX, TEX);
770    NV50_IR_OPCODE_CASE(TXD, TXD);
771    NV50_IR_OPCODE_CASE(TXP, TEX);
772 
773    NV50_IR_OPCODE_CASE(CAL, CALL);
774    NV50_IR_OPCODE_CASE(RET, RET);
775    NV50_IR_OPCODE_CASE(CMP, SLCT);
776 
777    NV50_IR_OPCODE_CASE(TXB, TXB);
778 
779    NV50_IR_OPCODE_CASE(DIV, DIV);
780 
781    NV50_IR_OPCODE_CASE(TXL, TXL);
782    NV50_IR_OPCODE_CASE(TEX_LZ, TXL);
783 
784    NV50_IR_OPCODE_CASE(CEIL, CEIL);
785    NV50_IR_OPCODE_CASE(I2F, CVT);
786    NV50_IR_OPCODE_CASE(NOT, NOT);
787    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
788    NV50_IR_OPCODE_CASE(SHL, SHL);
789 
790    NV50_IR_OPCODE_CASE(AND, AND);
791    NV50_IR_OPCODE_CASE(OR, OR);
792    NV50_IR_OPCODE_CASE(MOD, MOD);
793    NV50_IR_OPCODE_CASE(XOR, XOR);
794    NV50_IR_OPCODE_CASE(TXF, TXF);
795    NV50_IR_OPCODE_CASE(TXF_LZ, TXF);
796    NV50_IR_OPCODE_CASE(TXQ, TXQ);
797    NV50_IR_OPCODE_CASE(TXQS, TXQ);
798    NV50_IR_OPCODE_CASE(TG4, TXG);
799    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
800 
801    NV50_IR_OPCODE_CASE(EMIT, EMIT);
802    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
803 
804    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
805 
806    NV50_IR_OPCODE_CASE(F2I, CVT);
807    NV50_IR_OPCODE_CASE(FSEQ, SET);
808    NV50_IR_OPCODE_CASE(FSGE, SET);
809    NV50_IR_OPCODE_CASE(FSLT, SET);
810    NV50_IR_OPCODE_CASE(FSNE, SET);
811    NV50_IR_OPCODE_CASE(IDIV, DIV);
812    NV50_IR_OPCODE_CASE(IMAX, MAX);
813    NV50_IR_OPCODE_CASE(IMIN, MIN);
814    NV50_IR_OPCODE_CASE(IABS, ABS);
815    NV50_IR_OPCODE_CASE(INEG, NEG);
816    NV50_IR_OPCODE_CASE(ISGE, SET);
817    NV50_IR_OPCODE_CASE(ISHR, SHR);
818    NV50_IR_OPCODE_CASE(ISLT, SET);
819    NV50_IR_OPCODE_CASE(F2U, CVT);
820    NV50_IR_OPCODE_CASE(U2F, CVT);
821    NV50_IR_OPCODE_CASE(UADD, ADD);
822    NV50_IR_OPCODE_CASE(UDIV, DIV);
823    NV50_IR_OPCODE_CASE(UMAD, MAD);
824    NV50_IR_OPCODE_CASE(UMAX, MAX);
825    NV50_IR_OPCODE_CASE(UMIN, MIN);
826    NV50_IR_OPCODE_CASE(UMOD, MOD);
827    NV50_IR_OPCODE_CASE(UMUL, MUL);
828    NV50_IR_OPCODE_CASE(USEQ, SET);
829    NV50_IR_OPCODE_CASE(USGE, SET);
830    NV50_IR_OPCODE_CASE(USHR, SHR);
831    NV50_IR_OPCODE_CASE(USLT, SET);
832    NV50_IR_OPCODE_CASE(USNE, SET);
833 
834    NV50_IR_OPCODE_CASE(DABS, ABS);
835    NV50_IR_OPCODE_CASE(DNEG, NEG);
836    NV50_IR_OPCODE_CASE(DADD, ADD);
837    NV50_IR_OPCODE_CASE(DMUL, MUL);
838    NV50_IR_OPCODE_CASE(DDIV, DIV);
839    NV50_IR_OPCODE_CASE(DMAX, MAX);
840    NV50_IR_OPCODE_CASE(DMIN, MIN);
841    NV50_IR_OPCODE_CASE(DSLT, SET);
842    NV50_IR_OPCODE_CASE(DSGE, SET);
843    NV50_IR_OPCODE_CASE(DSEQ, SET);
844    NV50_IR_OPCODE_CASE(DSNE, SET);
845    NV50_IR_OPCODE_CASE(DRCP, RCP);
846    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
847    NV50_IR_OPCODE_CASE(DMAD, MAD);
848    NV50_IR_OPCODE_CASE(DFMA, FMA);
849    NV50_IR_OPCODE_CASE(D2I, CVT);
850    NV50_IR_OPCODE_CASE(D2U, CVT);
851    NV50_IR_OPCODE_CASE(I2D, CVT);
852    NV50_IR_OPCODE_CASE(U2D, CVT);
853    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
854    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
855    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
856    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
857    NV50_IR_OPCODE_CASE(DROUND, CVT);
858 
859    NV50_IR_OPCODE_CASE(U64SEQ, SET);
860    NV50_IR_OPCODE_CASE(U64SNE, SET);
861    NV50_IR_OPCODE_CASE(U64SLT, SET);
862    NV50_IR_OPCODE_CASE(U64SGE, SET);
863    NV50_IR_OPCODE_CASE(I64SLT, SET);
864    NV50_IR_OPCODE_CASE(I64SGE, SET);
865    NV50_IR_OPCODE_CASE(I2I64, CVT);
866    NV50_IR_OPCODE_CASE(U2I64, CVT);
867    NV50_IR_OPCODE_CASE(F2I64, CVT);
868    NV50_IR_OPCODE_CASE(F2U64, CVT);
869    NV50_IR_OPCODE_CASE(D2I64, CVT);
870    NV50_IR_OPCODE_CASE(D2U64, CVT);
871    NV50_IR_OPCODE_CASE(I642F, CVT);
872    NV50_IR_OPCODE_CASE(U642F, CVT);
873    NV50_IR_OPCODE_CASE(I642D, CVT);
874    NV50_IR_OPCODE_CASE(U642D, CVT);
875 
876    NV50_IR_OPCODE_CASE(I64MIN, MIN);
877    NV50_IR_OPCODE_CASE(U64MIN, MIN);
878    NV50_IR_OPCODE_CASE(I64MAX, MAX);
879    NV50_IR_OPCODE_CASE(U64MAX, MAX);
880    NV50_IR_OPCODE_CASE(I64ABS, ABS);
881    NV50_IR_OPCODE_CASE(I64NEG, NEG);
882    NV50_IR_OPCODE_CASE(U64ADD, ADD);
883    NV50_IR_OPCODE_CASE(U64MUL, MUL);
884    NV50_IR_OPCODE_CASE(U64SHL, SHL);
885    NV50_IR_OPCODE_CASE(I64SHR, SHR);
886    NV50_IR_OPCODE_CASE(U64SHR, SHR);
887 
888    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
889    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
890 
891    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
892    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
893    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
894    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
895    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
896    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
897    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
898    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
899    NV50_IR_OPCODE_CASE(GATHER4, TXG);
900    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
901 
902    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
903    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
904    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
905    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
906    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
907    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
908    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
909    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
910    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
911    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
912    NV50_IR_OPCODE_CASE(ATOMFADD, ATOM);
913    NV50_IR_OPCODE_CASE(ATOMDEC_WRAP, ATOM);
914    NV50_IR_OPCODE_CASE(ATOMINC_WRAP, ATOM);
915 
916    NV50_IR_OPCODE_CASE(TEX2, TEX);
917    NV50_IR_OPCODE_CASE(TXB2, TXB);
918    NV50_IR_OPCODE_CASE(TXL2, TXL);
919 
920    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
921    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
922    NV50_IR_OPCODE_CASE(BFI, INSBF);
923    NV50_IR_OPCODE_CASE(BREV, EXTBF);
924    NV50_IR_OPCODE_CASE(POPC, POPCNT);
925    NV50_IR_OPCODE_CASE(LSB, BFIND);
926    NV50_IR_OPCODE_CASE(IMSB, BFIND);
927    NV50_IR_OPCODE_CASE(UMSB, BFIND);
928 
929    NV50_IR_OPCODE_CASE(VOTE_ALL, VOTE);
930    NV50_IR_OPCODE_CASE(VOTE_ANY, VOTE);
931    NV50_IR_OPCODE_CASE(VOTE_EQ, VOTE);
932 
933    NV50_IR_OPCODE_CASE(BALLOT, VOTE);
934    NV50_IR_OPCODE_CASE(READ_INVOC, SHFL);
935    NV50_IR_OPCODE_CASE(READ_FIRST, SHFL);
936 
937    NV50_IR_OPCODE_CASE(END, EXIT);
938 
939    default:
940       return nv50_ir::OP_NOP;
941    }
942 }
943 
opcodeToSubOp(uint opcode)944 static uint16_t opcodeToSubOp(uint opcode)
945 {
946    switch (opcode) {
947    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
948    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
949    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
950    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
951    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
952    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
953    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
954    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
955    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
956    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
957    case TGSI_OPCODE_ATOMFADD: return NV50_IR_SUBOP_ATOM_ADD;
958    case TGSI_OPCODE_ATOMDEC_WRAP: return NV50_IR_SUBOP_ATOM_DEC;
959    case TGSI_OPCODE_ATOMINC_WRAP: return NV50_IR_SUBOP_ATOM_INC;
960    case TGSI_OPCODE_IMUL_HI:
961    case TGSI_OPCODE_UMUL_HI:
962       return NV50_IR_SUBOP_MUL_HIGH;
963    case TGSI_OPCODE_VOTE_ALL: return NV50_IR_SUBOP_VOTE_ALL;
964    case TGSI_OPCODE_VOTE_ANY: return NV50_IR_SUBOP_VOTE_ANY;
965    case TGSI_OPCODE_VOTE_EQ: return NV50_IR_SUBOP_VOTE_UNI;
966    default:
967       return 0;
968    }
969 }
970 
checkDstSrcAliasing() const971 bool Instruction::checkDstSrcAliasing() const
972 {
973    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
974       return false;
975 
976    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
977       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
978          break;
979       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
980           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
981          return true;
982    }
983    return false;
984 }
985 
986 class Source
987 {
988 public:
989    Source(struct nv50_ir_prog_info *, struct nv50_ir_prog_info_out *, nv50_ir::Program *);
990    ~Source();
991 
992 public:
993    bool scanSource();
fileSize(unsigned file) const994    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
995 
996 public:
997    struct tgsi_shader_info scan;
998    struct tgsi_full_instruction *insns;
999    const struct tgsi_token *tokens;
1000    struct nv50_ir_prog_info *info;
1001    struct nv50_ir_prog_info_out *info_out;
1002 
1003    nv50_ir::DynArray tempArrays;
1004    nv50_ir::DynArray immdArrays;
1005 
1006    typedef nv50_ir::BuildUtil::Location Location;
1007    // these registers are per-subroutine, cannot be used for parameter passing
1008    std::set<Location> locals;
1009 
1010    std::set<int> indirectTempArrays;
1011    std::map<int, int> indirectTempOffsets;
1012    std::map<int, std::pair<int, int> > tempArrayInfo;
1013    std::vector<int> tempArrayId;
1014 
1015    int clipVertexOutput;
1016 
1017    struct TextureView {
1018       uint8_t target; // TGSI_TEXTURE_*
1019    };
1020    std::vector<TextureView> textureViews;
1021 
1022    /*
1023    struct Resource {
1024       uint8_t target; // TGSI_TEXTURE_*
1025       bool raw;
1026       uint8_t slot; // $surface index
1027    };
1028    std::vector<Resource> resources;
1029    */
1030 
1031    struct MemoryFile {
1032       uint8_t mem_type; // TGSI_MEMORY_TYPE_*
1033    };
1034    std::vector<MemoryFile> memoryFiles;
1035 
1036    std::vector<bool> bufferAtomics;
1037 
1038    struct {
1039       uint16_t count;   /* count of inline immediates */
1040       uint32_t *data;   /* inline immediate data */
1041    } immd;
1042 
1043 private:
1044    nv50_ir::Program *prog;
1045    int inferSysValDirection(unsigned sn) const;
1046    bool scanDeclaration(const struct tgsi_full_declaration *);
1047    bool scanInstruction(const struct tgsi_full_instruction *);
1048    void scanInstructionSrc(const Instruction& insn,
1049                            const Instruction::SrcRegister& src,
1050                            unsigned mask);
1051    void scanProperty(const struct tgsi_full_property *);
1052    void scanImmediate(const struct tgsi_full_immediate *);
1053 
1054    inline bool isEdgeFlagPassthrough(const Instruction&) const;
1055 };
1056 
Source(struct nv50_ir_prog_info * info,struct nv50_ir_prog_info_out * info_out,nv50_ir::Program * prog)1057 Source::Source(struct nv50_ir_prog_info *info, struct nv50_ir_prog_info_out *info_out,
1058                nv50_ir::Program *prog)
1059 :  insns(NULL), info(info), info_out(info_out), clipVertexOutput(-1), prog(prog)
1060 {
1061    tokens = (const struct tgsi_token *)info->bin.source;
1062 
1063    if (info->dbgFlags & NV50_IR_DEBUG_BASIC)
1064       tgsi_dump(tokens, 0);
1065 
1066    tgsi_scan_shader(tokens, &scan);
1067 
1068    immd.count = 0;
1069    immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
1070 }
1071 
~Source()1072 Source::~Source()
1073 {
1074    if (insns)
1075       FREE(insns);
1076 
1077    if (immd.data)
1078       FREE(immd.data);
1079 }
1080 
scanSource()1081 bool Source::scanSource()
1082 {
1083    unsigned insnCount = 0;
1084    struct tgsi_parse_context parse;
1085 
1086    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
1087                                                   sizeof(insns[0]));
1088    if (!insns)
1089       return false;
1090 
1091    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
1092    //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
1093    tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
1094    memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
1095    bufferAtomics.resize(scan.file_max[TGSI_FILE_BUFFER] + 1);
1096 
1097    info_out->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
1098    info_out->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
1099    info_out->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
1100 
1101    if (info->type == PIPE_SHADER_FRAGMENT) {
1102       info_out->prop.fp.writesDepth = scan.writes_z;
1103       info_out->prop.fp.usesDiscard = scan.uses_kill || info->io.alphaRefBase;
1104    } else
1105    if (info->type == PIPE_SHADER_GEOMETRY) {
1106       info_out->prop.gp.instanceCount = 1; // default value
1107    }
1108 
1109    info->io.viewportId = -1;
1110 
1111    tgsi_parse_init(&parse, tokens);
1112    while (!tgsi_parse_end_of_tokens(&parse)) {
1113       tgsi_parse_token(&parse);
1114 
1115       switch (parse.FullToken.Token.Type) {
1116       case TGSI_TOKEN_TYPE_IMMEDIATE:
1117          scanImmediate(&parse.FullToken.FullImmediate);
1118          break;
1119       case TGSI_TOKEN_TYPE_DECLARATION:
1120          scanDeclaration(&parse.FullToken.FullDeclaration);
1121          break;
1122       case TGSI_TOKEN_TYPE_INSTRUCTION:
1123          insns[insnCount++] = parse.FullToken.FullInstruction;
1124          scanInstruction(&parse.FullToken.FullInstruction);
1125          break;
1126       case TGSI_TOKEN_TYPE_PROPERTY:
1127          scanProperty(&parse.FullToken.FullProperty);
1128          break;
1129       default:
1130          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
1131          break;
1132       }
1133    }
1134    tgsi_parse_free(&parse);
1135 
1136    if (indirectTempArrays.size()) {
1137       int tempBase = 0;
1138       for (std::set<int>::const_iterator it = indirectTempArrays.begin();
1139            it != indirectTempArrays.end(); ++it) {
1140          std::pair<int, int>& info = tempArrayInfo[*it];
1141          indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
1142          tempBase += info.second;
1143       }
1144       info_out->bin.tlsSpace += tempBase * 16;
1145    }
1146 
1147    if (info_out->io.genUserClip > 0) {
1148       info_out->io.clipDistances = info_out->io.genUserClip;
1149 
1150       const unsigned int nOut = (info_out->io.genUserClip + 3) / 4;
1151 
1152       for (unsigned int n = 0; n < nOut; ++n) {
1153          unsigned int i = info_out->numOutputs++;
1154          info_out->out[i].id = i;
1155          info_out->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
1156          info_out->out[i].si = n;
1157          info_out->out[i].mask = ((1 << info_out->io.clipDistances) - 1) >> (n * 4);
1158       }
1159    }
1160 
1161    return info->assignSlots(info_out) == 0;
1162 }
1163 
scanProperty(const struct tgsi_full_property * prop)1164 void Source::scanProperty(const struct tgsi_full_property *prop)
1165 {
1166    switch (prop->Property.PropertyName) {
1167    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
1168       info_out->prop.gp.outputPrim = prop->u[0].Data;
1169       break;
1170    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
1171       info_out->prop.gp.maxVertices = prop->u[0].Data;
1172       break;
1173    case TGSI_PROPERTY_GS_INVOCATIONS:
1174       info_out->prop.gp.instanceCount = prop->u[0].Data;
1175       break;
1176    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1177       info_out->prop.fp.separateFragData = true;
1178       break;
1179    case TGSI_PROPERTY_FS_COORD_ORIGIN:
1180    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1181    case TGSI_PROPERTY_FS_DEPTH_LAYOUT:
1182    case TGSI_PROPERTY_GS_INPUT_PRIM:
1183       // we don't care
1184       break;
1185    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1186       info_out->io.genUserClip = -1;
1187       break;
1188    case TGSI_PROPERTY_TCS_VERTICES_OUT:
1189       info_out->prop.tp.outputPatchSize = prop->u[0].Data;
1190       break;
1191    case TGSI_PROPERTY_TES_PRIM_MODE:
1192       info_out->prop.tp.domain = prop->u[0].Data;
1193       break;
1194    case TGSI_PROPERTY_TES_SPACING:
1195       info_out->prop.tp.partitioning = prop->u[0].Data;
1196       break;
1197    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1198       info_out->prop.tp.winding = prop->u[0].Data;
1199       break;
1200    case TGSI_PROPERTY_TES_POINT_MODE:
1201       if (prop->u[0].Data)
1202          info_out->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1203       else
1204          info_out->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1205       break;
1206    case TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH:
1207       info->prop.cp.numThreads[0] = prop->u[0].Data;
1208       break;
1209    case TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT:
1210       info->prop.cp.numThreads[1] = prop->u[0].Data;
1211       break;
1212    case TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH:
1213       info->prop.cp.numThreads[2] = prop->u[0].Data;
1214       break;
1215    case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1216       info_out->io.clipDistances = prop->u[0].Data;
1217       break;
1218    case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1219       info_out->io.cullDistances = prop->u[0].Data;
1220       break;
1221    case TGSI_PROPERTY_NEXT_SHADER:
1222       /* Do not need to know the next shader stage. */
1223       break;
1224    case TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL:
1225       info_out->prop.fp.earlyFragTests = prop->u[0].Data;
1226       break;
1227    case TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE:
1228       info_out->prop.fp.postDepthCoverage = prop->u[0].Data;
1229       break;
1230    case TGSI_PROPERTY_MUL_ZERO_WINS:
1231       info->io.mul_zero_wins = prop->u[0].Data;
1232       break;
1233    case TGSI_PROPERTY_LAYER_VIEWPORT_RELATIVE:
1234       info_out->io.layer_viewport_relative = prop->u[0].Data;
1235       break;
1236    default:
1237       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1238       break;
1239    }
1240 }
1241 
scanImmediate(const struct tgsi_full_immediate * imm)1242 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1243 {
1244    const unsigned n = immd.count++;
1245 
1246    assert(n < scan.immediate_count);
1247 
1248    for (int c = 0; c < 4; ++c)
1249       immd.data[n * 4 + c] = imm->u[c].Uint;
1250 }
1251 
inferSysValDirection(unsigned sn) const1252 int Source::inferSysValDirection(unsigned sn) const
1253 {
1254    switch (sn) {
1255    case TGSI_SEMANTIC_INSTANCEID:
1256    case TGSI_SEMANTIC_VERTEXID:
1257       return 1;
1258    case TGSI_SEMANTIC_LAYER:
1259 #if 0
1260    case TGSI_SEMANTIC_VIEWPORTINDEX:
1261       return 0;
1262 #endif
1263    case TGSI_SEMANTIC_PRIMID:
1264       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1265    default:
1266       return 0;
1267    }
1268 }
1269 
scanDeclaration(const struct tgsi_full_declaration * decl)1270 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1271 {
1272    unsigned i, c;
1273    unsigned sn = TGSI_SEMANTIC_GENERIC;
1274    unsigned si = 0;
1275    const unsigned first = decl->Range.First, last = decl->Range.Last;
1276    const int arrayId = decl->Array.ArrayID;
1277 
1278    if (decl->Declaration.Semantic) {
1279       sn = decl->Semantic.Name;
1280       si = decl->Semantic.Index;
1281    }
1282 
1283    if (decl->Declaration.Local || decl->Declaration.File == TGSI_FILE_ADDRESS) {
1284       for (i = first; i <= last; ++i) {
1285          for (c = 0; c < 4; ++c) {
1286             locals.insert(
1287                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1288          }
1289       }
1290    }
1291 
1292    switch (decl->Declaration.File) {
1293    case TGSI_FILE_INPUT:
1294       if (info->type == PIPE_SHADER_VERTEX) {
1295          // all vertex attributes are equal
1296          for (i = first; i <= last; ++i) {
1297             info_out->in[i].sn = TGSI_SEMANTIC_GENERIC;
1298             info_out->in[i].si = i;
1299          }
1300       } else {
1301          for (i = first; i <= last; ++i, ++si) {
1302             info_out->in[i].id = i;
1303             info_out->in[i].sn = sn;
1304             info_out->in[i].si = si;
1305             if (info->type == PIPE_SHADER_FRAGMENT) {
1306                // translate interpolation mode
1307                switch (decl->Interp.Interpolate) {
1308                case TGSI_INTERPOLATE_CONSTANT:
1309                   info_out->in[i].flat = 1;
1310                   break;
1311                case TGSI_INTERPOLATE_COLOR:
1312                   info_out->in[i].sc = 1;
1313                   break;
1314                case TGSI_INTERPOLATE_LINEAR:
1315                   info_out->in[i].linear = 1;
1316                   break;
1317                default:
1318                   break;
1319                }
1320                if (decl->Interp.Location)
1321                   info_out->in[i].centroid = 1;
1322             }
1323 
1324             if (sn == TGSI_SEMANTIC_PATCH)
1325                info_out->in[i].patch = 1;
1326             if (sn == TGSI_SEMANTIC_PATCH)
1327                info_out->numPatchConstants = MAX2(info_out->numPatchConstants, si + 1);
1328          }
1329       }
1330       break;
1331    case TGSI_FILE_OUTPUT:
1332       for (i = first; i <= last; ++i, ++si) {
1333          switch (sn) {
1334          case TGSI_SEMANTIC_POSITION:
1335             if (info->type == PIPE_SHADER_FRAGMENT)
1336                info_out->io.fragDepth = i;
1337             else
1338             if (clipVertexOutput < 0)
1339                clipVertexOutput = i;
1340             break;
1341          case TGSI_SEMANTIC_COLOR:
1342             if (info->type == PIPE_SHADER_FRAGMENT)
1343                info_out->prop.fp.numColourResults++;
1344             break;
1345          case TGSI_SEMANTIC_EDGEFLAG:
1346             info_out->io.edgeFlagOut = i;
1347             break;
1348          case TGSI_SEMANTIC_CLIPVERTEX:
1349             clipVertexOutput = i;
1350             break;
1351          case TGSI_SEMANTIC_CLIPDIST:
1352             info_out->io.genUserClip = -1;
1353             break;
1354          case TGSI_SEMANTIC_SAMPLEMASK:
1355             info_out->io.sampleMask = i;
1356             break;
1357          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1358             info->io.viewportId = i;
1359             break;
1360          case TGSI_SEMANTIC_PATCH:
1361             info_out->numPatchConstants = MAX2(info_out->numPatchConstants, si + 1);
1362             /* fallthrough */
1363          case TGSI_SEMANTIC_TESSOUTER:
1364          case TGSI_SEMANTIC_TESSINNER:
1365             info_out->out[i].patch = 1;
1366             break;
1367          default:
1368             break;
1369          }
1370          info_out->out[i].id = i;
1371          info_out->out[i].sn = sn;
1372          info_out->out[i].si = si;
1373       }
1374       break;
1375    case TGSI_FILE_SYSTEM_VALUE:
1376       switch (sn) {
1377       case TGSI_SEMANTIC_INSTANCEID:
1378          info_out->io.instanceId = first;
1379          break;
1380       case TGSI_SEMANTIC_VERTEXID:
1381          info_out->io.vertexId = first;
1382          break;
1383       case TGSI_SEMANTIC_BASEVERTEX:
1384       case TGSI_SEMANTIC_BASEINSTANCE:
1385       case TGSI_SEMANTIC_DRAWID:
1386          info_out->prop.vp.usesDrawParameters = true;
1387          break;
1388       case TGSI_SEMANTIC_SAMPLEID:
1389       case TGSI_SEMANTIC_SAMPLEPOS:
1390          prog->persampleInvocation = true;
1391          break;
1392       case TGSI_SEMANTIC_SAMPLEMASK:
1393          info_out->prop.fp.usesSampleMaskIn = true;
1394          break;
1395       default:
1396          break;
1397       }
1398       for (i = first; i <= last; ++i, ++si) {
1399          info_out->sv[i].sn = sn;
1400          info_out->sv[i].si = si;
1401          info_out->sv[i].input = inferSysValDirection(sn);
1402 
1403          switch (sn) {
1404          case TGSI_SEMANTIC_TESSOUTER:
1405          case TGSI_SEMANTIC_TESSINNER:
1406             info_out->sv[i].patch = 1;
1407             break;
1408          }
1409       }
1410       break;
1411 /*
1412    case TGSI_FILE_RESOURCE:
1413       for (i = first; i <= last; ++i) {
1414          resources[i].target = decl->Resource.Resource;
1415          resources[i].raw = decl->Resource.Raw;
1416          resources[i].slot = i;
1417       }
1418       break;
1419 */
1420    case TGSI_FILE_SAMPLER_VIEW:
1421       for (i = first; i <= last; ++i)
1422          textureViews[i].target = decl->SamplerView.Resource;
1423       break;
1424    case TGSI_FILE_MEMORY:
1425       for (i = first; i <= last; ++i)
1426          memoryFiles[i].mem_type = decl->Declaration.MemType;
1427       break;
1428    case TGSI_FILE_NULL:
1429    case TGSI_FILE_TEMPORARY:
1430       for (i = first; i <= last; ++i)
1431          tempArrayId[i] = arrayId;
1432       if (arrayId)
1433          tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1434                                                    first, last - first + 1)));
1435       break;
1436    case TGSI_FILE_BUFFER:
1437       for (i = first; i <= last; ++i)
1438          bufferAtomics[i] = decl->Declaration.Atomic;
1439       break;
1440    case TGSI_FILE_ADDRESS:
1441    case TGSI_FILE_CONSTANT:
1442    case TGSI_FILE_IMMEDIATE:
1443    case TGSI_FILE_SAMPLER:
1444    case TGSI_FILE_IMAGE:
1445       break;
1446    default:
1447       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1448       return false;
1449    }
1450    return true;
1451 }
1452 
isEdgeFlagPassthrough(const Instruction & insn) const1453 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1454 {
1455    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1456       insn.getDst(0).getIndex(0) == info_out->io.edgeFlagOut &&
1457       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1458 }
1459 
scanInstructionSrc(const Instruction & insn,const Instruction::SrcRegister & src,unsigned mask)1460 void Source::scanInstructionSrc(const Instruction& insn,
1461                                 const Instruction::SrcRegister& src,
1462                                 unsigned mask)
1463 {
1464    if (src.getFile() == TGSI_FILE_TEMPORARY) {
1465       if (src.isIndirect(0))
1466          indirectTempArrays.insert(src.getArrayId());
1467    } else
1468    if (src.getFile() == TGSI_FILE_OUTPUT) {
1469       if (src.isIndirect(0)) {
1470          // We don't know which one is accessed, just mark everything for
1471          // reading. This is an extremely unlikely occurrence.
1472          for (unsigned i = 0; i < info_out->numOutputs; ++i)
1473             info_out->out[i].oread = 1;
1474       } else {
1475          info_out->out[src.getIndex(0)].oread = 1;
1476       }
1477    }
1478    if (src.getFile() == TGSI_FILE_SYSTEM_VALUE) {
1479       if (info_out->sv[src.getIndex(0)].sn == TGSI_SEMANTIC_SAMPLEPOS)
1480          info_out->prop.fp.readsSampleLocations = true;
1481    }
1482    if (src.getFile() != TGSI_FILE_INPUT)
1483       return;
1484 
1485    if (src.isIndirect(0)) {
1486       for (unsigned i = 0; i < info_out->numInputs; ++i)
1487          info_out->in[i].mask = 0xf;
1488    } else {
1489       const int i = src.getIndex(0);
1490       for (unsigned c = 0; c < 4; ++c) {
1491          if (!(mask & (1 << c)))
1492             continue;
1493          int k = src.getSwizzle(c);
1494          if (k <= TGSI_SWIZZLE_W)
1495             info_out->in[i].mask |= 1 << k;
1496       }
1497       switch (info_out->in[i].sn) {
1498       case TGSI_SEMANTIC_PSIZE:
1499       case TGSI_SEMANTIC_PRIMID:
1500       case TGSI_SEMANTIC_FOG:
1501          info_out->in[i].mask &= 0x1;
1502          break;
1503       case TGSI_SEMANTIC_PCOORD:
1504          info_out->in[i].mask &= 0x3;
1505          break;
1506       default:
1507          break;
1508       }
1509    }
1510 }
1511 
scanInstruction(const struct tgsi_full_instruction * inst)1512 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1513 {
1514    Instruction insn(inst);
1515 
1516    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1517       info_out->numBarriers = 1;
1518 
1519    if (insn.getOpcode() == TGSI_OPCODE_FBFETCH)
1520       info_out->prop.fp.readsFramebuffer = true;
1521 
1522    if (insn.getOpcode() == TGSI_OPCODE_INTERP_SAMPLE)
1523       info_out->prop.fp.readsSampleLocations = true;
1524 
1525    if (insn.getOpcode() == TGSI_OPCODE_DEMOTE)
1526       info_out->prop.fp.usesDiscard = true;
1527 
1528    if (insn.dstCount()) {
1529       Instruction::DstRegister dst = insn.getDst(0);
1530 
1531       if (insn.getOpcode() == TGSI_OPCODE_STORE &&
1532           dst.getFile() != TGSI_FILE_MEMORY) {
1533          info_out->io.globalAccess |= 0x2;
1534 
1535          if (dst.getFile() == TGSI_FILE_INPUT) {
1536             // TODO: Handle indirect somehow?
1537             const int i = dst.getIndex(0);
1538             info_out->in[i].mask |= 1;
1539          }
1540       }
1541 
1542       if (dst.getFile() == TGSI_FILE_OUTPUT) {
1543          if (dst.isIndirect(0))
1544             for (unsigned i = 0; i < info_out->numOutputs; ++i)
1545                info_out->out[i].mask = 0xf;
1546          else
1547             info_out->out[dst.getIndex(0)].mask |= dst.getMask();
1548 
1549          if (info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1550              info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1551              info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1552              info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1553              info_out->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1554             info_out->out[dst.getIndex(0)].mask &= 1;
1555 
1556          if (isEdgeFlagPassthrough(insn))
1557             info_out->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1558       } else
1559       if (dst.getFile() == TGSI_FILE_TEMPORARY) {
1560          if (dst.isIndirect(0))
1561             indirectTempArrays.insert(dst.getArrayId());
1562       } else
1563       if (dst.getFile() == TGSI_FILE_BUFFER ||
1564           dst.getFile() == TGSI_FILE_IMAGE ||
1565           (dst.getFile() == TGSI_FILE_MEMORY &&
1566            memoryFiles[dst.getIndex(0)].mem_type == TGSI_MEMORY_TYPE_GLOBAL)) {
1567          info_out->io.globalAccess |= 0x2;
1568       }
1569    }
1570 
1571    if (insn.srcCount() && (
1572              insn.getSrc(0).getFile() != TGSI_FILE_MEMORY ||
1573              memoryFiles[insn.getSrc(0).getIndex(0)].mem_type ==
1574              TGSI_MEMORY_TYPE_GLOBAL)) {
1575       switch (insn.getOpcode()) {
1576       case TGSI_OPCODE_ATOMUADD:
1577       case TGSI_OPCODE_ATOMXCHG:
1578       case TGSI_OPCODE_ATOMCAS:
1579       case TGSI_OPCODE_ATOMAND:
1580       case TGSI_OPCODE_ATOMOR:
1581       case TGSI_OPCODE_ATOMXOR:
1582       case TGSI_OPCODE_ATOMUMIN:
1583       case TGSI_OPCODE_ATOMIMIN:
1584       case TGSI_OPCODE_ATOMUMAX:
1585       case TGSI_OPCODE_ATOMIMAX:
1586       case TGSI_OPCODE_ATOMFADD:
1587       case TGSI_OPCODE_ATOMDEC_WRAP:
1588       case TGSI_OPCODE_ATOMINC_WRAP:
1589       case TGSI_OPCODE_LOAD:
1590          info_out->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1591             0x1 : 0x2;
1592          break;
1593       }
1594    }
1595 
1596 
1597    for (unsigned s = 0; s < insn.srcCount(); ++s)
1598       scanInstructionSrc(insn, insn.getSrc(s), insn.srcMask(s));
1599 
1600    for (unsigned s = 0; s < insn.getNumTexOffsets(); ++s)
1601       scanInstructionSrc(insn, insn.getTexOffset(s), insn.texOffsetMask());
1602 
1603    return true;
1604 }
1605 
1606 nv50_ir::TexInstruction::Target
getTexture(const tgsi::Source * code,int s) const1607 Instruction::getTexture(const tgsi::Source *code, int s) const
1608 {
1609    // XXX: indirect access
1610    unsigned int r;
1611 
1612    switch (getSrc(s).getFile()) {
1613 /*
1614    case TGSI_FILE_RESOURCE:
1615       r = getSrc(s).getIndex(0);
1616       return translateTexture(code->resources.at(r).target);
1617 */
1618    case TGSI_FILE_SAMPLER_VIEW:
1619       r = getSrc(s).getIndex(0);
1620       return translateTexture(code->textureViews.at(r).target);
1621    default:
1622       return translateTexture(insn->Texture.Texture);
1623    }
1624 }
1625 
1626 } // namespace tgsi
1627 
1628 namespace {
1629 
1630 using namespace nv50_ir;
1631 
1632 class Converter : public ConverterCommon
1633 {
1634 public:
1635    Converter(Program *, const tgsi::Source *, nv50_ir_prog_info_out *);
1636    ~Converter();
1637 
1638    bool run();
1639 
1640 private:
1641    Value *shiftAddress(Value *);
1642    Value *getVertexBase(int s);
1643    Value *getOutputBase(int s);
1644    DataArray *getArrayForFile(unsigned file, int idx);
1645    Value *fetchSrc(int s, int c);
1646    Value *fetchDst(int d, int c);
1647    Value *acquireDst(int d, int c);
1648    void storeDst(int d, int c, Value *);
1649 
1650    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1651    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1652                  Value *val, Value *ptr);
1653 
1654    void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1655    Value *applySrcMod(Value *, int s, int c);
1656 
1657    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1658    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1659    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1660 
1661    bool isSubGroupMask(uint8_t semantic);
1662 
1663    bool handleInstruction(const struct tgsi_full_instruction *);
1664    void exportOutputs();
1665    inline bool isEndOfSubroutine(uint ip);
1666 
1667    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1668 
1669    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1670    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1671    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1672    void handleTXF(Value *dst0[4], int R, int L_M);
1673    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1674    void handleFBFETCH(Value *dst0[4]);
1675    void handleLIT(Value *dst0[4]);
1676 
1677    // Symbol *getResourceBase(int r);
1678    void getImageCoords(std::vector<Value *>&, int s);
1679 
1680    void handleLOAD(Value *dst0[4]);
1681    void handleSTORE();
1682    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1683 
1684    void handleINTERP(Value *dst0[4]);
1685 
1686    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1687 
1688    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1689 
1690    Value *buildDot(int dim);
1691 
1692    class BindArgumentsPass : public Pass {
1693    public:
BindArgumentsPass(Converter & conv)1694       BindArgumentsPass(Converter &conv) : conv(conv) { }
1695 
1696    private:
1697       Converter &conv;
1698       Subroutine *sub;
1699 
1700       inline const Location *getValueLocation(Subroutine *, Value *);
1701 
1702       template<typename T> inline void
1703       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1704                      T (Function::*proto));
1705 
1706       template<typename T> inline void
1707       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1708                       T (Function::*proto));
1709 
1710    protected:
1711       bool visit(Function *);
visit(BasicBlock * bb)1712       bool visit(BasicBlock *bb) { return false; }
1713    };
1714 
1715 private:
1716    const tgsi::Source *code;
1717 
1718    uint ip; // instruction pointer
1719 
1720    tgsi::Instruction tgsi;
1721 
1722    DataType dstTy;
1723    DataType srcTy;
1724 
1725    DataArray tData; // TGSI_FILE_TEMPORARY
1726    DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1727    DataArray aData; // TGSI_FILE_ADDRESS
1728    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1729 
1730    Value *zero;
1731 
1732    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1733    uint8_t vtxBaseValid;
1734 
1735    Stack condBBs;  // fork BB, then else clause BB
1736    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1737    Stack loopBBs;  // loop headers
1738    Stack breakBBs; // end of / after loop
1739 
1740    Value *viewport;
1741 };
1742 
1743 Symbol *
srcToSym(tgsi::Instruction::SrcRegister src,int c)1744 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1745 {
1746    const int swz = src.getSwizzle(c);
1747 
1748    /* TODO: Use Array ID when it's available for the index */
1749    return makeSym(src.getFile(),
1750                   src.is2D() ? src.getIndex(1) : 0,
1751                   src.getIndex(0), swz,
1752                   src.getIndex(0) * 16 + swz * 4);
1753 }
1754 
1755 Symbol *
dstToSym(tgsi::Instruction::DstRegister dst,int c)1756 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1757 {
1758    /* TODO: Use Array ID when it's available for the index */
1759    return makeSym(dst.getFile(),
1760                   dst.is2D() ? dst.getIndex(1) : 0,
1761                   dst.getIndex(0), c,
1762                   dst.getIndex(0) * 16 + c * 4);
1763 }
1764 
1765 Symbol *
makeSym(uint tgsiFile,int fileIdx,int idx,int c,uint32_t address)1766 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1767 {
1768    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1769 
1770    sym->reg.fileIndex = fileIdx;
1771 
1772    if (tgsiFile == TGSI_FILE_MEMORY) {
1773       switch (code->memoryFiles[fileIdx].mem_type) {
1774       case TGSI_MEMORY_TYPE_GLOBAL:
1775          /* No-op this is the default for TGSI_FILE_MEMORY */
1776          sym->setFile(FILE_MEMORY_GLOBAL);
1777          break;
1778       case TGSI_MEMORY_TYPE_SHARED:
1779          sym->setFile(FILE_MEMORY_SHARED);
1780          break;
1781       case TGSI_MEMORY_TYPE_INPUT:
1782          assert(prog->getType() == Program::TYPE_COMPUTE);
1783          assert(idx == -1);
1784          sym->setFile(FILE_SHADER_INPUT);
1785          address += info->prop.cp.inputOffset;
1786          break;
1787       default:
1788          assert(0); /* TODO: Add support for global and private memory */
1789       }
1790    }
1791 
1792    if (idx >= 0) {
1793       if (sym->reg.file == FILE_SHADER_INPUT)
1794          sym->setOffset(info_out->in[idx].slot[c] * 4);
1795       else
1796       if (sym->reg.file == FILE_SHADER_OUTPUT)
1797          sym->setOffset(info_out->out[idx].slot[c] * 4);
1798       else
1799       if (sym->reg.file == FILE_SYSTEM_VALUE)
1800          sym->setSV(tgsi::translateSysVal(info_out->sv[idx].sn), c);
1801       else
1802          sym->setOffset(address);
1803    } else {
1804       sym->setOffset(address);
1805    }
1806    return sym;
1807 }
1808 
1809 Value *
interpolate(tgsi::Instruction::SrcRegister src,int c,Value * ptr)1810 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1811 {
1812    operation op;
1813 
1814    // XXX: no way to know interpolation mode if we don't know what's accessed
1815    const uint8_t mode = translateInterpMode(&info_out->in[ptr ? 0 :
1816                                                       src.getIndex(0)], op);
1817 
1818    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1819 
1820    insn->setDef(0, getScratch());
1821    insn->setSrc(0, srcToSym(src, c));
1822    if (op == OP_PINTERP)
1823       insn->setSrc(1, fragCoord[3]);
1824    if (ptr)
1825       insn->setIndirect(0, 0, ptr);
1826 
1827    insn->setInterpolate(mode);
1828 
1829    bb->insertTail(insn);
1830    return insn->getDef(0);
1831 }
1832 
1833 Value *
applySrcMod(Value * val,int s,int c)1834 Converter::applySrcMod(Value *val, int s, int c)
1835 {
1836    Modifier m = tgsi.getSrc(s).getMod(c);
1837    DataType ty = tgsi.inferSrcType();
1838 
1839    if (m & Modifier(NV50_IR_MOD_ABS))
1840       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1841 
1842    if (m & Modifier(NV50_IR_MOD_NEG))
1843       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1844 
1845    return val;
1846 }
1847 
1848 Value *
getVertexBase(int s)1849 Converter::getVertexBase(int s)
1850 {
1851    assert(s < 5);
1852    if (!(vtxBaseValid & (1 << s))) {
1853       const int index = tgsi.getSrc(s).getIndex(1);
1854       Value *rel = NULL;
1855       if (tgsi.getSrc(s).isIndirect(1))
1856          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1857       vtxBaseValid |= 1 << s;
1858       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1859                           mkImm(index), rel);
1860    }
1861    return vtxBase[s];
1862 }
1863 
1864 Value *
getOutputBase(int s)1865 Converter::getOutputBase(int s)
1866 {
1867    assert(s < 5);
1868    if (!(vtxBaseValid & (1 << s))) {
1869       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1870       if (tgsi.getSrc(s).isIndirect(1))
1871          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1872                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1873                          offset);
1874       vtxBaseValid |= 1 << s;
1875       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1876    }
1877    return vtxBase[s];
1878 }
1879 
1880 Value *
fetchSrc(int s,int c)1881 Converter::fetchSrc(int s, int c)
1882 {
1883    Value *res;
1884    Value *ptr = NULL, *dimRel = NULL;
1885 
1886    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1887 
1888    if (src.isIndirect(0))
1889       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1890 
1891    if (src.is2D()) {
1892       switch (src.getFile()) {
1893       case TGSI_FILE_OUTPUT:
1894          dimRel = getOutputBase(s);
1895          break;
1896       case TGSI_FILE_INPUT:
1897          dimRel = getVertexBase(s);
1898          break;
1899       case TGSI_FILE_CONSTANT:
1900          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1901          if (src.isIndirect(1))
1902             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1903          break;
1904       default:
1905          break;
1906       }
1907    }
1908 
1909    res = fetchSrc(src, c, ptr);
1910 
1911    if (dimRel)
1912       res->getInsn()->setIndirect(0, 1, dimRel);
1913 
1914    return applySrcMod(res, s, c);
1915 }
1916 
1917 Value *
fetchDst(int d,int c)1918 Converter::fetchDst(int d, int c)
1919 {
1920    Value *res;
1921    Value *ptr = NULL, *dimRel = NULL;
1922 
1923    tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1924 
1925    if (dst.isIndirect(0))
1926       ptr = fetchSrc(dst.getIndirect(0), 0, NULL);
1927 
1928    if (dst.is2D()) {
1929       switch (dst.getFile()) {
1930       case TGSI_FILE_OUTPUT:
1931          assert(0); // TODO
1932          dimRel = NULL;
1933          break;
1934       case TGSI_FILE_INPUT:
1935          assert(0); // TODO
1936          dimRel = NULL;
1937          break;
1938       case TGSI_FILE_CONSTANT:
1939          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1940          if (dst.isIndirect(1))
1941             dimRel = fetchSrc(dst.getIndirect(1), 0, 0);
1942          break;
1943       default:
1944          break;
1945       }
1946    }
1947 
1948    struct tgsi_full_src_register fsr = dst.asSrc();
1949    tgsi::Instruction::SrcRegister src(&fsr);
1950    res = fetchSrc(src, c, ptr);
1951 
1952    if (dimRel)
1953       res->getInsn()->setIndirect(0, 1, dimRel);
1954 
1955    return res;
1956 }
1957 
1958 Converter::DataArray *
getArrayForFile(unsigned file,int idx)1959 Converter::getArrayForFile(unsigned file, int idx)
1960 {
1961    switch (file) {
1962    case TGSI_FILE_TEMPORARY:
1963       return idx == 0 ? &tData : &lData;
1964    case TGSI_FILE_ADDRESS:
1965       return &aData;
1966    case TGSI_FILE_OUTPUT:
1967       assert(prog->getType() == Program::TYPE_FRAGMENT);
1968       return &oData;
1969    default:
1970       assert(!"invalid/unhandled TGSI source file");
1971       return NULL;
1972    }
1973 }
1974 
1975 Value *
shiftAddress(Value * index)1976 Converter::shiftAddress(Value *index)
1977 {
1978    if (!index)
1979       return NULL;
1980    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1981 }
1982 
1983 void
adjustTempIndex(int arrayId,int & idx,int & idx2d) const1984 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1985 {
1986    std::map<int, int>::const_iterator it =
1987       code->indirectTempOffsets.find(arrayId);
1988    if (it == code->indirectTempOffsets.end())
1989       return;
1990 
1991    idx2d = 1;
1992    idx += it->second;
1993 }
1994 
1995 bool
isSubGroupMask(uint8_t semantic)1996 Converter::isSubGroupMask(uint8_t semantic)
1997 {
1998    switch (semantic) {
1999       case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2000       case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2001       case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2002       case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2003       case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2004          return true;
2005       default:
2006          return false;
2007    }
2008 }
2009 
2010 Value *
fetchSrc(tgsi::Instruction::SrcRegister src,int c,Value * ptr)2011 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
2012 {
2013    int idx2d = src.is2D() ? src.getIndex(1) : 0;
2014    int idx = src.getIndex(0);
2015    const int swz = src.getSwizzle(c);
2016    Instruction *ld;
2017 
2018    switch (src.getFile()) {
2019    case TGSI_FILE_IMMEDIATE:
2020       assert(!ptr);
2021       return loadImm(NULL, code->immd.data[idx * 4 + swz]);
2022    case TGSI_FILE_CONSTANT:
2023       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
2024    case TGSI_FILE_INPUT:
2025       if (prog->getType() == Program::TYPE_FRAGMENT) {
2026          // don't load masked inputs, won't be assigned a slot
2027          if (!ptr && !(info_out->in[idx].mask & (1 << swz)))
2028             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
2029          return interpolate(src, c, shiftAddress(ptr));
2030       } else
2031       if (prog->getType() == Program::TYPE_GEOMETRY) {
2032          if (!ptr && info_out->in[idx].sn == TGSI_SEMANTIC_PRIMID)
2033             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
2034          // XXX: This is going to be a problem with scalar arrays, i.e. when
2035          // we cannot assume that the address is given in units of vec4.
2036          //
2037          // nv50 and nvc0 need different things here, so let the lowering
2038          // passes decide what to do with the address
2039          if (ptr)
2040             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
2041       }
2042       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2043       ld->perPatch = info_out->in[idx].patch;
2044       return ld->getDef(0);
2045    case TGSI_FILE_OUTPUT:
2046       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2047       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2048       ld->perPatch = info_out->out[idx].patch;
2049       return ld->getDef(0);
2050    case TGSI_FILE_SYSTEM_VALUE:
2051       assert(!ptr);
2052       if (info_out->sv[idx].sn == TGSI_SEMANTIC_THREAD_ID &&
2053           info->prop.cp.numThreads[swz] == 1)
2054          return loadImm(NULL, 0u);
2055       if (isSubGroupMask(info_out->sv[idx].sn) && swz > 0)
2056          return loadImm(NULL, 0u);
2057       if (info_out->sv[idx].sn == TGSI_SEMANTIC_SUBGROUP_SIZE)
2058          return loadImm(NULL, 32u);
2059       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
2060       ld->perPatch = info_out->sv[idx].patch;
2061       return ld->getDef(0);
2062    case TGSI_FILE_TEMPORARY: {
2063       int arrayid = src.getArrayId();
2064       if (!arrayid)
2065          arrayid = code->tempArrayId[idx];
2066       adjustTempIndex(arrayid, idx, idx2d);
2067    }
2068       /* fallthrough */
2069    default:
2070       return getArrayForFile(src.getFile(), idx2d)->load(
2071          sub.cur->values, idx, swz, shiftAddress(ptr));
2072    }
2073 }
2074 
2075 Value *
acquireDst(int d,int c)2076 Converter::acquireDst(int d, int c)
2077 {
2078    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2079    const unsigned f = dst.getFile();
2080    int idx = dst.getIndex(0);
2081    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2082 
2083    if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY ||
2084        f == TGSI_FILE_IMAGE)
2085       return NULL;
2086 
2087    if (dst.isIndirect(0) ||
2088        f == TGSI_FILE_SYSTEM_VALUE ||
2089        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
2090       return getScratch();
2091 
2092    if (f == TGSI_FILE_TEMPORARY) {
2093       int arrayid = dst.getArrayId();
2094       if (!arrayid)
2095          arrayid = code->tempArrayId[idx];
2096       adjustTempIndex(arrayid, idx, idx2d);
2097    }
2098 
2099    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
2100 }
2101 
2102 void
storeDst(int d,int c,Value * val)2103 Converter::storeDst(int d, int c, Value *val)
2104 {
2105    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2106 
2107    if (tgsi.getSaturate()) {
2108       mkOp1(OP_SAT, dstTy, val, val);
2109    }
2110 
2111    Value *ptr = NULL;
2112    if (dst.isIndirect(0))
2113       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
2114 
2115    if (info_out->io.genUserClip > 0 &&
2116        dst.getFile() == TGSI_FILE_OUTPUT &&
2117        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
2118       mkMov(clipVtx[c], val);
2119       val = clipVtx[c];
2120    }
2121 
2122    storeDst(dst, c, val, ptr);
2123 }
2124 
2125 void
storeDst(const tgsi::Instruction::DstRegister dst,int c,Value * val,Value * ptr)2126 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
2127                     Value *val, Value *ptr)
2128 {
2129    const unsigned f = dst.getFile();
2130    int idx = dst.getIndex(0);
2131    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2132 
2133    if (f == TGSI_FILE_SYSTEM_VALUE) {
2134       assert(!ptr);
2135       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
2136    } else
2137    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
2138 
2139       if (ptr || (info_out->out[idx].mask & (1 << c))) {
2140          /* Save the viewport index into a scratch register so that it can be
2141             exported at EMIT time */
2142          if (info_out->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
2143              prog->getType() == Program::TYPE_GEOMETRY &&
2144              viewport != NULL)
2145             mkOp1(OP_MOV, TYPE_U32, viewport, val);
2146          else
2147             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
2148                info_out->out[idx].patch;
2149       }
2150    } else
2151    if (f == TGSI_FILE_TEMPORARY ||
2152        f == TGSI_FILE_ADDRESS ||
2153        f == TGSI_FILE_OUTPUT) {
2154       if (f == TGSI_FILE_TEMPORARY) {
2155          int arrayid = dst.getArrayId();
2156          if (!arrayid)
2157             arrayid = code->tempArrayId[idx];
2158          adjustTempIndex(arrayid, idx, idx2d);
2159       }
2160 
2161       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
2162    } else {
2163       assert(!"invalid dst file");
2164    }
2165 }
2166 
2167 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
2168    for (chan = 0; chan < 4; ++chan)                 \
2169       if (!inst.getDst(d).isMasked(chan))
2170 
2171 Value *
buildDot(int dim)2172 Converter::buildDot(int dim)
2173 {
2174    assert(dim > 0);
2175 
2176    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
2177    Value *dotp = getScratch();
2178 
2179    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1)
2180       ->dnz = info->io.mul_zero_wins;
2181 
2182    for (int c = 1; c < dim; ++c) {
2183       src0 = fetchSrc(0, c);
2184       src1 = fetchSrc(1, c);
2185       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp)
2186          ->dnz = info->io.mul_zero_wins;
2187    }
2188    return dotp;
2189 }
2190 
2191 void
insertConvergenceOps(BasicBlock * conv,BasicBlock * fork)2192 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
2193 {
2194    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
2195    join->fixed = 1;
2196    conv->insertHead(join);
2197 
2198    assert(!fork->joinAt);
2199    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
2200    fork->insertBefore(fork->getExit(), fork->joinAt);
2201 }
2202 
2203 void
setTexRS(TexInstruction * tex,unsigned int & s,int R,int S)2204 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
2205 {
2206    unsigned rIdx = 0, sIdx = 0;
2207 
2208    if (R >= 0 && tgsi.getSrc(R).getFile() != TGSI_FILE_SAMPLER) {
2209       // This is the bindless case. We have to get the actual value and pass
2210       // it in. This will be the complete handle.
2211       tex->tex.rIndirectSrc = s;
2212       tex->setSrc(s++, fetchSrc(R, 0));
2213       tex->setTexture(tgsi.getTexture(code, R), 0xff, 0x1f);
2214       tex->tex.bindless = true;
2215       return;
2216    }
2217 
2218    if (R >= 0)
2219       rIdx = tgsi.getSrc(R).getIndex(0);
2220    if (S >= 0)
2221       sIdx = tgsi.getSrc(S).getIndex(0);
2222 
2223    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
2224 
2225    if (tgsi.getSrc(R).isIndirect(0)) {
2226       tex->tex.rIndirectSrc = s;
2227       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
2228    }
2229    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
2230       tex->tex.sIndirectSrc = s;
2231       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
2232    }
2233 }
2234 
2235 void
handleTXQ(Value * dst0[4],enum TexQuery query,int R)2236 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
2237 {
2238    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
2239    tex->tex.query = query;
2240    unsigned int c, d;
2241 
2242    for (d = 0, c = 0; c < 4; ++c) {
2243       if (!dst0[c])
2244          continue;
2245       tex->tex.mask |= 1 << c;
2246       tex->setDef(d++, dst0[c]);
2247    }
2248    if (query == TXQ_DIMS)
2249       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
2250    else
2251       tex->setSrc((c = 0), zero);
2252 
2253    setTexRS(tex, ++c, R, -1);
2254 
2255    bb->insertTail(tex);
2256 }
2257 
2258 void
loadProjTexCoords(Value * dst[4],Value * src[4],unsigned int mask)2259 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
2260 {
2261    Value *proj = fetchSrc(0, 3);
2262    Instruction *insn = proj->getUniqueInsn();
2263    int c;
2264 
2265    if (insn->op == OP_PINTERP) {
2266       bb->insertTail(insn = cloneForward(func, insn));
2267       insn->op = OP_LINTERP;
2268       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
2269       insn->setSrc(1, NULL);
2270       proj = insn->getDef(0);
2271    }
2272    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
2273 
2274    for (c = 0; c < 4; ++c) {
2275       if (!(mask & (1 << c)))
2276          continue;
2277       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
2278          continue;
2279       mask &= ~(1 << c);
2280 
2281       bb->insertTail(insn = cloneForward(func, insn));
2282       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
2283       insn->setSrc(1, proj);
2284       dst[c] = insn->getDef(0);
2285    }
2286    if (!mask)
2287       return;
2288 
2289    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
2290 
2291    for (c = 0; c < 4; ++c)
2292       if (mask & (1 << c))
2293          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
2294 }
2295 
2296 // order of nv50 ir sources: x y z layer lod/bias shadow
2297 // order of TGSI TEX sources: x y z layer shadow lod/bias
2298 //  lowering will finally set the hw specific order (like array first on nvc0)
2299 void
handleTEX(Value * dst[4],int R,int S,int L,int C,int Dx,int Dy)2300 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
2301 {
2302    Value *arg[4], *src[8];
2303    Value *lod = NULL, *shd = NULL;
2304    unsigned int s, c, d;
2305    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2306 
2307    TexInstruction::Target tgt = tgsi.getTexture(code, R);
2308 
2309    for (s = 0; s < tgt.getArgCount(); ++s)
2310       arg[s] = src[s] = fetchSrc(0, s);
2311 
2312    if (tgsi.getOpcode() == TGSI_OPCODE_TEX_LZ)
2313       lod = loadImm(NULL, 0);
2314    else if (texi->op == OP_TXL || texi->op == OP_TXB)
2315       lod = fetchSrc(L >> 4, L & 3);
2316 
2317    if (C == 0x0f)
2318       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2319 
2320    if (tgt == TEX_TARGET_CUBE_ARRAY_SHADOW) {
2321       switch (tgsi.getOpcode()) {
2322       case TGSI_OPCODE_TG4: shd = fetchSrc(1, 0); break;
2323       case TGSI_OPCODE_TEX2: shd = fetchSrc(1, 0); break;
2324       case TGSI_OPCODE_TXB2: shd = fetchSrc(1, 1); break;
2325       case TGSI_OPCODE_TXL2: shd = fetchSrc(1, 1); break;
2326       default: assert(!"unexpected opcode with cube array shadow"); break;
2327       }
2328    }
2329    else if (tgt.isShadow())
2330       shd = fetchSrc(C >> 4, C & 3);
2331 
2332    if (texi->op == OP_TXD) {
2333       for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2334          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2335          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2336       }
2337    }
2338 
2339    // cube textures don't care about projection value, it's divided out
2340    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2341       unsigned int n = tgt.getDim();
2342       if (shd) {
2343          arg[n] = shd;
2344          ++n;
2345          assert(tgt.getDim() == tgt.getArgCount());
2346       }
2347       loadProjTexCoords(src, arg, (1 << n) - 1);
2348       if (shd)
2349          shd = src[n - 1];
2350    }
2351 
2352    for (c = 0, d = 0; c < 4; ++c) {
2353       if (dst[c]) {
2354          texi->setDef(d++, dst[c]);
2355          texi->tex.mask |= 1 << c;
2356       } else {
2357          // NOTE: maybe hook up def too, for CSE
2358       }
2359    }
2360    for (s = 0; s < tgt.getArgCount(); ++s)
2361       texi->setSrc(s, src[s]);
2362    if (lod)
2363       texi->setSrc(s++, lod);
2364    if (shd)
2365       texi->setSrc(s++, shd);
2366 
2367    setTexRS(texi, s, R, S);
2368 
2369    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2370       texi->tex.levelZero = true;
2371    if (prog->getType() != Program::TYPE_FRAGMENT &&
2372        (tgsi.getOpcode() == TGSI_OPCODE_TEX ||
2373         tgsi.getOpcode() == TGSI_OPCODE_TEX2 ||
2374         tgsi.getOpcode() == TGSI_OPCODE_TXP))
2375       texi->tex.levelZero = true;
2376    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2377       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, code->immd.data);
2378 
2379    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2380    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2381       for (c = 0; c < 3; ++c) {
2382          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2383          texi->offset[s][c].setInsn(texi);
2384       }
2385    }
2386 
2387    bb->insertTail(texi);
2388 }
2389 
2390 // 1st source: xyz = coordinates, w = lod/sample
2391 // 2nd source: offset
2392 void
handleTXF(Value * dst[4],int R,int L_M)2393 Converter::handleTXF(Value *dst[4], int R, int L_M)
2394 {
2395    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2396    int ms;
2397    unsigned int c, d, s;
2398 
2399    texi->tex.target = tgsi.getTexture(code, R);
2400 
2401    ms = texi->tex.target.isMS() ? 1 : 0;
2402    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2403 
2404    for (c = 0, d = 0; c < 4; ++c) {
2405       if (dst[c]) {
2406          texi->setDef(d++, dst[c]);
2407          texi->tex.mask |= 1 << c;
2408       }
2409    }
2410    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2411       texi->setSrc(c, fetchSrc(0, c));
2412    if (!ms && tgsi.getOpcode() == TGSI_OPCODE_TXF_LZ)
2413       texi->setSrc(c++, loadImm(NULL, 0));
2414    else
2415       texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2416 
2417    setTexRS(texi, c, R, -1);
2418 
2419    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2420    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2421       for (c = 0; c < 3; ++c) {
2422          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2423          texi->offset[s][c].setInsn(texi);
2424       }
2425    }
2426 
2427    bb->insertTail(texi);
2428 }
2429 
2430 void
handleFBFETCH(Value * dst[4])2431 Converter::handleFBFETCH(Value *dst[4])
2432 {
2433    TexInstruction *texi = new_TexInstruction(func, OP_TXF);
2434    unsigned int c, d;
2435 
2436    texi->tex.target = TEX_TARGET_2D_MS_ARRAY;
2437    texi->tex.levelZero = 1;
2438    texi->tex.useOffsets = 0;
2439 
2440    for (c = 0, d = 0; c < 4; ++c) {
2441       if (dst[c]) {
2442          texi->setDef(d++, dst[c]);
2443          texi->tex.mask |= 1 << c;
2444       }
2445    }
2446 
2447    Value *x = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 0));
2448    Value *y = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 1));
2449    Value *z = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_LAYER, 0));
2450    Value *ms = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_SAMPLE_INDEX, 0));
2451 
2452    mkCvt(OP_CVT, TYPE_U32, x, TYPE_F32, x)->rnd = ROUND_Z;
2453    mkCvt(OP_CVT, TYPE_U32, y, TYPE_F32, y)->rnd = ROUND_Z;
2454    texi->setSrc(0, x);
2455    texi->setSrc(1, y);
2456    texi->setSrc(2, z);
2457    texi->setSrc(3, ms);
2458 
2459    texi->tex.r = texi->tex.s = -1;
2460 
2461    bb->insertTail(texi);
2462 }
2463 
2464 void
handleLIT(Value * dst0[4])2465 Converter::handleLIT(Value *dst0[4])
2466 {
2467    Value *val0 = NULL;
2468    unsigned int mask = tgsi.getDst(0).getMask();
2469 
2470    if (mask & (1 << 0))
2471       loadImm(dst0[0], 1.0f);
2472 
2473    if (mask & (1 << 3))
2474       loadImm(dst0[3], 1.0f);
2475 
2476    if (mask & (3 << 1)) {
2477       val0 = getScratch();
2478       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2479       if (mask & (1 << 1))
2480          mkMov(dst0[1], val0);
2481    }
2482 
2483    if (mask & (1 << 2)) {
2484       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2485       Value *val1 = getScratch(), *val3 = getScratch();
2486 
2487       Value *pos128 = loadImm(NULL, +127.999999f);
2488       Value *neg128 = loadImm(NULL, -127.999999f);
2489 
2490       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2491       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2492       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2493       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2494 
2495       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2496    }
2497 }
2498 
2499 /* Keep this around for now as reference when adding img support
2500 static inline bool
2501 isResourceSpecial(const int r)
2502 {
2503    return (r == TGSI_RESOURCE_GLOBAL ||
2504            r == TGSI_RESOURCE_LOCAL ||
2505            r == TGSI_RESOURCE_PRIVATE ||
2506            r == TGSI_RESOURCE_INPUT);
2507 }
2508 
2509 static inline bool
2510 isResourceRaw(const tgsi::Source *code, const int r)
2511 {
2512    return isResourceSpecial(r) || code->resources[r].raw;
2513 }
2514 
2515 static inline nv50_ir::TexTarget
2516 getResourceTarget(const tgsi::Source *code, int r)
2517 {
2518    if (isResourceSpecial(r))
2519       return nv50_ir::TEX_TARGET_BUFFER;
2520    return tgsi::translateTexture(code->resources.at(r).target);
2521 }
2522 
2523 Symbol *
2524 Converter::getResourceBase(const int r)
2525 {
2526    Symbol *sym = NULL;
2527 
2528    switch (r) {
2529    case TGSI_RESOURCE_GLOBAL:
2530       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL,
2531                        info->io.auxCBSlot);
2532       break;
2533    case TGSI_RESOURCE_LOCAL:
2534       assert(prog->getType() == Program::TYPE_COMPUTE);
2535       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2536                      info->prop.cp.sharedOffset);
2537       break;
2538    case TGSI_RESOURCE_PRIVATE:
2539       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2540                      info->bin.tlsSpace);
2541       break;
2542    case TGSI_RESOURCE_INPUT:
2543       assert(prog->getType() == Program::TYPE_COMPUTE);
2544       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2545                      info->prop.cp.inputOffset);
2546       break;
2547    default:
2548       sym = new_Symbol(prog,
2549                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2550       break;
2551    }
2552    return sym;
2553 }
2554 
2555 void
2556 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2557 {
2558    const int arg =
2559       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2560 
2561    for (int c = 0; c < arg; ++c)
2562       coords.push_back(fetchSrc(s, c));
2563 
2564    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2565    if (r == TGSI_RESOURCE_LOCAL ||
2566        r == TGSI_RESOURCE_PRIVATE ||
2567        r == TGSI_RESOURCE_INPUT)
2568       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2569                          coords[0]);
2570 }
2571 
2572 static inline int
2573 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2574 {
2575    int n = 0;
2576 
2577    while (mask) {
2578       if (mask & 1) {
2579          size[n]++;
2580       } else {
2581          if (size[n])
2582             comp[n = 1] = size[0] + 1;
2583          else
2584             comp[n]++;
2585       }
2586       mask >>= 1;
2587    }
2588    if (size[0] == 3) {
2589       n = 1;
2590       size[0] = (comp[0] == 1) ? 1 : 2;
2591       size[1] = 3 - size[0];
2592       comp[1] = comp[0] + size[0];
2593    }
2594    return n + 1;
2595 }
2596 */
2597 void
getImageCoords(std::vector<Value * > & coords,int s)2598 Converter::getImageCoords(std::vector<Value *> &coords, int s)
2599 {
2600    TexInstruction::Target t =
2601       TexInstruction::Target(tgsi.getImageTarget());
2602    const int arg = t.getDim() + (t.isArray() || t.isCube());
2603 
2604    for (int c = 0; c < arg; ++c)
2605       coords.push_back(fetchSrc(s, c));
2606 
2607    if (t.isMS())
2608       coords.push_back(fetchSrc(s, 3));
2609 }
2610 
2611 // For raw loads, granularity is 4 byte.
2612 // Usage of the texture read mask on OP_SULDP is not allowed.
2613 void
handleLOAD(Value * dst0[4])2614 Converter::handleLOAD(Value *dst0[4])
2615 {
2616    const int r = tgsi.getSrc(0).getIndex(0);
2617    int c;
2618    std::vector<Value *> off, src, ldv, def;
2619    Value *ind = NULL;
2620 
2621    if (tgsi.getSrc(0).isIndirect(0))
2622       ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2623 
2624    switch (tgsi.getSrc(0).getFile()) {
2625    case TGSI_FILE_BUFFER:
2626    case TGSI_FILE_MEMORY:
2627       for (c = 0; c < 4; ++c) {
2628          if (!dst0[c])
2629             continue;
2630 
2631          Value *off;
2632          Symbol *sym;
2633          uint32_t src0_component_offset = tgsi.getSrc(0).getSwizzle(c) * 4;
2634 
2635          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2636             off = NULL;
2637             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2638                           tgsi.getSrc(1).getValueU32(0, code->immd.data) +
2639                           src0_component_offset);
2640          } else {
2641             // yzw are ignored for buffers
2642             off = fetchSrc(1, 0);
2643             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2644                           src0_component_offset);
2645          }
2646 
2647          Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2648          if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER &&
2649              code->bufferAtomics[r])
2650             ld->cache = nv50_ir::CACHE_CG;
2651          else
2652             ld->cache = tgsi.getCacheMode();
2653          if (ind)
2654             ld->setIndirect(0, 1, ind);
2655       }
2656       break;
2657    default: {
2658       getImageCoords(off, 1);
2659       def.resize(4);
2660 
2661       for (c = 0; c < 4; ++c) {
2662          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2663             def[c] = getScratch();
2664          else
2665             def[c] = dst0[c];
2666       }
2667 
2668       bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2669       if (bindless)
2670          ind = fetchSrc(0, 0);
2671 
2672       TexInstruction *ld =
2673          mkTex(OP_SULDP, tgsi.getImageTarget(), 0, 0, def, off);
2674       ld->tex.mask = tgsi.getDst(0).getMask();
2675       ld->tex.format = tgsi.getImageFormat();
2676       ld->cache = tgsi.getCacheMode();
2677       ld->tex.bindless = bindless;
2678       if (!bindless)
2679          ld->tex.r = r;
2680       if (ind)
2681          ld->setIndirectR(ind);
2682 
2683       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2684          if (dst0[c] != def[c])
2685             mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2686       break;
2687    }
2688    }
2689 
2690 
2691 /* Keep this around for now as reference when adding img support
2692    getResourceCoords(off, r, 1);
2693 
2694    if (isResourceRaw(code, r)) {
2695       uint8_t mask = 0;
2696       uint8_t comp[2] = { 0, 0 };
2697       uint8_t size[2] = { 0, 0 };
2698 
2699       Symbol *base = getResourceBase(r);
2700 
2701       // determine the base and size of the at most 2 load ops
2702       for (c = 0; c < 4; ++c)
2703          if (!tgsi.getDst(0).isMasked(c))
2704             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2705 
2706       int n = partitionLoadStore(comp, size, mask);
2707 
2708       src = off;
2709 
2710       def.resize(4); // index by component, the ones we need will be non-NULL
2711       for (c = 0; c < 4; ++c) {
2712          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2713             def[c] = dst0[c];
2714          else
2715          if (mask & (1 << c))
2716             def[c] = getScratch();
2717       }
2718 
2719       const bool useLd = isResourceSpecial(r) ||
2720          (info->io.nv50styleSurfaces &&
2721           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2722 
2723       for (int i = 0; i < n; ++i) {
2724          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2725 
2726          if (comp[i]) // adjust x component of source address if necessary
2727             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2728                             off[0], mkImm(comp[i] * 4));
2729          else
2730             src[0] = off[0];
2731 
2732          if (useLd) {
2733             Instruction *ld =
2734                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2735             for (size_t c = 1; c < ldv.size(); ++c)
2736                ld->setDef(c, ldv[c]);
2737          } else {
2738             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2739                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2740          }
2741       }
2742    } else {
2743       def.resize(4);
2744       for (c = 0; c < 4; ++c) {
2745          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2746             def[c] = getScratch();
2747          else
2748             def[c] = dst0[c];
2749       }
2750 
2751       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2752             def, off);
2753    }
2754    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2755       if (dst0[c] != def[c])
2756          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2757 */
2758 }
2759 
2760 // For formatted stores, the write mask on OP_SUSTP can be used.
2761 // Raw stores have to be split.
2762 void
handleSTORE()2763 Converter::handleSTORE()
2764 {
2765    const int r = tgsi.getDst(0).getIndex(0);
2766    int c;
2767    std::vector<Value *> off, src, dummy;
2768    Value *ind = NULL;
2769 
2770    if (tgsi.getDst(0).isIndirect(0))
2771       ind = fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0);
2772 
2773    switch (tgsi.getDst(0).getFile()) {
2774    case TGSI_FILE_BUFFER:
2775    case TGSI_FILE_MEMORY:
2776       for (c = 0; c < 4; ++c) {
2777          if (!(tgsi.getDst(0).getMask() & (1 << c)))
2778             continue;
2779 
2780          Symbol *sym;
2781          Value *off;
2782          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2783             off = NULL;
2784             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2785                           tgsi.getSrc(0).getValueU32(0, code->immd.data) + 4 * c);
2786          } else {
2787             // yzw are ignored for buffers
2788             off = fetchSrc(0, 0);
2789             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2790          }
2791 
2792          Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2793          st->cache = tgsi.getCacheMode();
2794          if (ind)
2795             st->setIndirect(0, 1, ind);
2796       }
2797       break;
2798    default: {
2799       getImageCoords(off, 0);
2800       src = off;
2801 
2802       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2803          src.push_back(fetchSrc(1, c));
2804 
2805       bool bindless = tgsi.getDst(0).getFile() != TGSI_FILE_IMAGE;
2806       if (bindless)
2807          ind = fetchDst(0, 0);
2808 
2809       TexInstruction *st =
2810          mkTex(OP_SUSTP, tgsi.getImageTarget(), 0, 0, dummy, src);
2811       st->tex.mask = tgsi.getDst(0).getMask();
2812       st->tex.format = tgsi.getImageFormat();
2813       st->cache = tgsi.getCacheMode();
2814       st->tex.bindless = bindless;
2815       if (!bindless)
2816          st->tex.r = r;
2817       if (ind)
2818          st->setIndirectR(ind);
2819 
2820       break;
2821    }
2822    }
2823 
2824 /* Keep this around for now as reference when adding img support
2825    getResourceCoords(off, r, 0);
2826    src = off;
2827    const int s = src.size();
2828 
2829    if (isResourceRaw(code, r)) {
2830       uint8_t comp[2] = { 0, 0 };
2831       uint8_t size[2] = { 0, 0 };
2832 
2833       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2834 
2835       Symbol *base = getResourceBase(r);
2836 
2837       const bool useSt = isResourceSpecial(r) ||
2838          (info->io.nv50styleSurfaces &&
2839           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2840 
2841       for (int i = 0; i < n; ++i) {
2842          if (comp[i]) // adjust x component of source address if necessary
2843             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2844                             off[0], mkImm(comp[i] * 4));
2845          else
2846             src[0] = off[0];
2847 
2848          const DataType stTy = typeOfSize(size[i] * 4);
2849 
2850          if (useSt) {
2851             Instruction *st =
2852                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2853             for (c = 1; c < size[i]; ++c)
2854                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2855             st->setIndirect(0, 0, src[0]);
2856          } else {
2857             // attach values to be stored
2858             src.resize(s + size[i]);
2859             for (c = 0; c < size[i]; ++c)
2860                src[s + c] = fetchSrc(1, comp[i] + c);
2861             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2862                   0, dummy, src)->setType(stTy);
2863          }
2864       }
2865    } else {
2866       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2867          src.push_back(fetchSrc(1, c));
2868 
2869       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2870             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2871    }
2872 */
2873 }
2874 
2875 // XXX: These only work on resources with the single-component u32/s32 formats.
2876 // Therefore the result is replicated. This might not be intended by TGSI, but
2877 // operating on more than 1 component would produce undefined results because
2878 // they do not exist.
2879 void
handleATOM(Value * dst0[4],DataType ty,uint16_t subOp)2880 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2881 {
2882    const int r = tgsi.getSrc(0).getIndex(0);
2883    std::vector<Value *> srcv;
2884    std::vector<Value *> defv;
2885    LValue *dst = getScratch();
2886    Value *ind = NULL;
2887 
2888    if (tgsi.getSrc(0).isIndirect(0))
2889       ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2890 
2891    switch (tgsi.getSrc(0).getFile()) {
2892    case TGSI_FILE_BUFFER:
2893    case TGSI_FILE_MEMORY:
2894       for (int c = 0; c < 4; ++c) {
2895          if (!dst0[c])
2896             continue;
2897 
2898          Instruction *insn;
2899          Value *off = fetchSrc(1, c);
2900          Value *sym;
2901          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2902             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2903                           tgsi.getSrc(1).getValueU32(c, code->immd.data));
2904          else
2905             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2906          if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2907             insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2908          else
2909             insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2910          if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2911             insn->setIndirect(0, 0, off);
2912          if (ind)
2913             insn->setIndirect(0, 1, ind);
2914          insn->subOp = subOp;
2915       }
2916       for (int c = 0; c < 4; ++c)
2917          if (dst0[c])
2918             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2919       break;
2920    default: {
2921       getImageCoords(srcv, 1);
2922       defv.push_back(dst);
2923       srcv.push_back(fetchSrc(2, 0));
2924 
2925       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2926          srcv.push_back(fetchSrc(3, 0));
2927 
2928       bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2929       if (bindless)
2930          ind = fetchSrc(0, 0);
2931 
2932       TexInstruction *tex = mkTex(OP_SUREDP, tgsi.getImageTarget(),
2933                                   0, 0, defv, srcv);
2934       tex->subOp = subOp;
2935       tex->tex.mask = 1;
2936       tex->tex.format = tgsi.getImageFormat();
2937       tex->setType(ty);
2938       tex->tex.bindless = bindless;
2939       if (!bindless)
2940          tex->tex.r = r;
2941       if (ind)
2942          tex->setIndirectR(ind);
2943 
2944       for (int c = 0; c < 4; ++c)
2945          if (dst0[c])
2946             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2947       break;
2948    }
2949    }
2950 
2951 /* Keep this around for now as reference when adding img support
2952    getResourceCoords(srcv, r, 1);
2953 
2954    if (isResourceSpecial(r)) {
2955       assert(r != TGSI_RESOURCE_INPUT);
2956       Instruction *insn;
2957       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2958       insn->subOp = subOp;
2959       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2960          insn->setSrc(2, fetchSrc(3, 0));
2961       insn->setIndirect(0, 0, srcv.at(0));
2962    } else {
2963       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2964       TexTarget targ = getResourceTarget(code, r);
2965       int idx = code->resources[r].slot;
2966       defv.push_back(dst);
2967       srcv.push_back(fetchSrc(2, 0));
2968       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2969          srcv.push_back(fetchSrc(3, 0));
2970       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2971       tex->subOp = subOp;
2972       tex->tex.mask = 1;
2973       tex->setType(ty);
2974    }
2975 
2976    for (int c = 0; c < 4; ++c)
2977       if (dst0[c])
2978          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2979 */
2980 }
2981 
2982 void
handleINTERP(Value * dst[4])2983 Converter::handleINTERP(Value *dst[4])
2984 {
2985    // Check whether the input is linear. All other attributes ignored.
2986    Instruction *insn;
2987    Value *offset = NULL, *ptr = NULL, *w = NULL;
2988    Symbol *sym[4] = { NULL };
2989    bool linear;
2990    operation op = OP_NOP;
2991    int c, mode = 0;
2992 
2993    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2994 
2995    // In some odd cases, in large part due to varying packing, the source
2996    // might not actually be an input. This is illegal TGSI, but it's easier to
2997    // account for it here than it is to fix it where the TGSI is being
2998    // generated. In that case, it's going to be a straight up mov (or sequence
2999    // of mov's) from the input in question. We follow the mov chain to see
3000    // which input we need to use.
3001    if (src.getFile() != TGSI_FILE_INPUT) {
3002       if (src.isIndirect(0)) {
3003          ERROR("Ignoring indirect input interpolation\n");
3004          return;
3005       }
3006       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3007          Value *val = fetchSrc(0, c);
3008          assert(val->defs.size() == 1);
3009          insn = val->getInsn();
3010          while (insn->op == OP_MOV) {
3011             assert(insn->getSrc(0)->defs.size() == 1);
3012             insn = insn->getSrc(0)->getInsn();
3013             if (!insn) {
3014                ERROR("Miscompiling shader due to unhandled INTERP\n");
3015                return;
3016             }
3017          }
3018          if (insn->op != OP_LINTERP && insn->op != OP_PINTERP) {
3019             ERROR("Trying to interpolate non-input, this is not allowed.\n");
3020             return;
3021          }
3022          sym[c] = insn->getSrc(0)->asSym();
3023          assert(sym[c]);
3024          op = insn->op;
3025          mode = insn->ipa;
3026          ptr = insn->getIndirect(0, 0);
3027       }
3028    } else {
3029       if (src.isIndirect(0))
3030          ptr = shiftAddress(fetchSrc(src.getIndirect(0), 0, NULL));
3031 
3032       // We can assume that the fixed index will point to an input of the same
3033       // interpolation type in case of an indirect.
3034       // TODO: Make use of ArrayID.
3035       linear = info_out->in[src.getIndex(0)].linear;
3036       if (linear) {
3037          op = OP_LINTERP;
3038          mode = NV50_IR_INTERP_LINEAR;
3039       } else {
3040          op = OP_PINTERP;
3041          mode = NV50_IR_INTERP_PERSPECTIVE;
3042       }
3043    }
3044 
3045    switch (tgsi.getOpcode()) {
3046    case TGSI_OPCODE_INTERP_CENTROID:
3047       mode |= NV50_IR_INTERP_CENTROID;
3048       break;
3049    case TGSI_OPCODE_INTERP_SAMPLE:
3050       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
3051       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
3052       mode |= NV50_IR_INTERP_OFFSET;
3053       break;
3054    case TGSI_OPCODE_INTERP_OFFSET: {
3055       // The input in src1.xy is float, but we need a single 32-bit value
3056       // where the upper and lower 16 bits are encoded in S0.12 format. We need
3057       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
3058       // and then convert to s32.
3059       Value *offs[2];
3060       for (c = 0; c < 2; c++) {
3061          offs[c] = getScratch();
3062          mkOp2(OP_MIN, TYPE_F32, offs[c], fetchSrc(1, c), loadImm(NULL, 0.4375f));
3063          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
3064          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
3065          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
3066       }
3067       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
3068                       offs[1], mkImm(0x1010), offs[0]);
3069       mode |= NV50_IR_INTERP_OFFSET;
3070       break;
3071    }
3072    }
3073 
3074    if (op == OP_PINTERP) {
3075       if (offset) {
3076          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
3077          mkOp1(OP_RCP, TYPE_F32, w, w);
3078       } else {
3079          w = fragCoord[3];
3080       }
3081    }
3082 
3083 
3084    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3085       insn = mkOp1(op, TYPE_F32, dst[c], sym[c] ? sym[c] : srcToSym(src, c));
3086       if (op == OP_PINTERP)
3087          insn->setSrc(1, w);
3088       if (offset)
3089          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
3090       if (ptr)
3091          insn->setIndirect(0, 0, ptr);
3092 
3093       insn->setInterpolate(mode);
3094    }
3095 }
3096 
3097 bool
isEndOfSubroutine(uint ip)3098 Converter::isEndOfSubroutine(uint ip)
3099 {
3100    assert(ip < code->scan.num_instructions);
3101    tgsi::Instruction insn(&code->insns[ip]);
3102    return (insn.getOpcode() == TGSI_OPCODE_END ||
3103            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
3104            // does END occur at end of main or the very end ?
3105            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
3106 }
3107 
3108 bool
handleInstruction(const struct tgsi_full_instruction * insn)3109 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
3110 {
3111    Instruction *geni;
3112 
3113    Value *dst0[4], *rDst0[4];
3114    Value *src0, *src1, *src2, *src3;
3115    Value *val0, *val1;
3116    int c;
3117 
3118    tgsi = tgsi::Instruction(insn);
3119 
3120    bool useScratchDst = tgsi.checkDstSrcAliasing();
3121 
3122    operation op = tgsi.getOP();
3123    dstTy = tgsi.inferDstType();
3124    srcTy = tgsi.inferSrcType();
3125 
3126    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
3127 
3128    if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
3129       for (c = 0; c < 4; ++c) {
3130          rDst0[c] = acquireDst(0, c);
3131          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
3132       }
3133    }
3134 
3135    switch (tgsi.getOpcode()) {
3136    case TGSI_OPCODE_ADD:
3137    case TGSI_OPCODE_UADD:
3138    case TGSI_OPCODE_AND:
3139    case TGSI_OPCODE_DIV:
3140    case TGSI_OPCODE_IDIV:
3141    case TGSI_OPCODE_UDIV:
3142    case TGSI_OPCODE_MAX:
3143    case TGSI_OPCODE_MIN:
3144    case TGSI_OPCODE_IMAX:
3145    case TGSI_OPCODE_IMIN:
3146    case TGSI_OPCODE_UMAX:
3147    case TGSI_OPCODE_UMIN:
3148    case TGSI_OPCODE_MOD:
3149    case TGSI_OPCODE_UMOD:
3150    case TGSI_OPCODE_MUL:
3151    case TGSI_OPCODE_UMUL:
3152    case TGSI_OPCODE_IMUL_HI:
3153    case TGSI_OPCODE_UMUL_HI:
3154    case TGSI_OPCODE_OR:
3155    case TGSI_OPCODE_SHL:
3156    case TGSI_OPCODE_ISHR:
3157    case TGSI_OPCODE_USHR:
3158    case TGSI_OPCODE_XOR:
3159       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3160          src0 = fetchSrc(0, c);
3161          src1 = fetchSrc(1, c);
3162          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
3163          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3164          if (op == OP_MUL && dstTy == TYPE_F32)
3165             geni->dnz = info->io.mul_zero_wins;
3166          geni->precise = insn->Instruction.Precise;
3167       }
3168       break;
3169    case TGSI_OPCODE_MAD:
3170    case TGSI_OPCODE_UMAD:
3171    case TGSI_OPCODE_FMA:
3172       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3173          src0 = fetchSrc(0, c);
3174          src1 = fetchSrc(1, c);
3175          src2 = fetchSrc(2, c);
3176          geni = mkOp3(op, dstTy, dst0[c], src0, src1, src2);
3177          if (dstTy == TYPE_F32)
3178             geni->dnz = info->io.mul_zero_wins;
3179          geni->precise = insn->Instruction.Precise;
3180       }
3181       break;
3182    case TGSI_OPCODE_MOV:
3183    case TGSI_OPCODE_CEIL:
3184    case TGSI_OPCODE_FLR:
3185    case TGSI_OPCODE_TRUNC:
3186    case TGSI_OPCODE_RCP:
3187    case TGSI_OPCODE_SQRT:
3188    case TGSI_OPCODE_IABS:
3189    case TGSI_OPCODE_INEG:
3190    case TGSI_OPCODE_NOT:
3191    case TGSI_OPCODE_DDX:
3192    case TGSI_OPCODE_DDY:
3193    case TGSI_OPCODE_DDX_FINE:
3194    case TGSI_OPCODE_DDY_FINE:
3195       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3196          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
3197       break;
3198    case TGSI_OPCODE_RSQ:
3199       src0 = fetchSrc(0, 0);
3200       val0 = getScratch();
3201       mkOp1(OP_ABS, TYPE_F32, val0, src0);
3202       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
3203       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3204          mkMov(dst0[c], val0);
3205       break;
3206    case TGSI_OPCODE_ARL:
3207    case TGSI_OPCODE_ARR:
3208       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3209          const RoundMode rnd =
3210             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
3211          src0 = fetchSrc(0, c);
3212          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
3213       }
3214       break;
3215    case TGSI_OPCODE_UARL:
3216       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3217          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
3218       break;
3219    case TGSI_OPCODE_POW:
3220       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
3221       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3222          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3223       break;
3224    case TGSI_OPCODE_EX2:
3225    case TGSI_OPCODE_LG2:
3226       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
3227       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3228          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3229       break;
3230    case TGSI_OPCODE_COS:
3231    case TGSI_OPCODE_SIN:
3232       val0 = getScratch();
3233       if (mask & 7) {
3234          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
3235          mkOp1(op, TYPE_F32, val0, val0);
3236          for (c = 0; c < 3; ++c)
3237             if (dst0[c])
3238                mkMov(dst0[c], val0);
3239       }
3240       if (dst0[3]) {
3241          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
3242          mkOp1(op, TYPE_F32, dst0[3], val0);
3243       }
3244       break;
3245    case TGSI_OPCODE_EXP:
3246       src0 = fetchSrc(0, 0);
3247       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
3248       if (dst0[1])
3249          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
3250       if (dst0[0])
3251          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
3252       if (dst0[2])
3253          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
3254       if (dst0[3])
3255          loadImm(dst0[3], 1.0f);
3256       break;
3257    case TGSI_OPCODE_LOG:
3258       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
3259       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
3260       if (dst0[0] || dst0[1])
3261          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
3262       if (dst0[1]) {
3263          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
3264          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
3265          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0)
3266             ->dnz = info->io.mul_zero_wins;
3267       }
3268       if (dst0[3])
3269          loadImm(dst0[3], 1.0f);
3270       break;
3271    case TGSI_OPCODE_DP2:
3272       val0 = buildDot(2);
3273       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3274          mkMov(dst0[c], val0);
3275       break;
3276    case TGSI_OPCODE_DP3:
3277       val0 = buildDot(3);
3278       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3279          mkMov(dst0[c], val0);
3280       break;
3281    case TGSI_OPCODE_DP4:
3282       val0 = buildDot(4);
3283       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3284          mkMov(dst0[c], val0);
3285       break;
3286    case TGSI_OPCODE_DST:
3287       if (dst0[0])
3288          loadImm(dst0[0], 1.0f);
3289       if (dst0[1]) {
3290          src0 = fetchSrc(0, 1);
3291          src1 = fetchSrc(1, 1);
3292          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1)
3293             ->dnz = info->io.mul_zero_wins;
3294       }
3295       if (dst0[2])
3296          mkMov(dst0[2], fetchSrc(0, 2));
3297       if (dst0[3])
3298          mkMov(dst0[3], fetchSrc(1, 3));
3299       break;
3300    case TGSI_OPCODE_LRP:
3301       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3302          src0 = fetchSrc(0, c);
3303          src1 = fetchSrc(1, c);
3304          src2 = fetchSrc(2, c);
3305          mkOp3(OP_MAD, TYPE_F32, dst0[c],
3306                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2)
3307             ->dnz = info->io.mul_zero_wins;
3308       }
3309       break;
3310    case TGSI_OPCODE_LIT:
3311       handleLIT(dst0);
3312       break;
3313    case TGSI_OPCODE_ISSG:
3314    case TGSI_OPCODE_SSG:
3315       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3316          src0 = fetchSrc(0, c);
3317          val0 = getScratch();
3318          val1 = getScratch();
3319          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
3320          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
3321          if (srcTy == TYPE_F32)
3322             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
3323          else
3324             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
3325       }
3326       break;
3327    case TGSI_OPCODE_UCMP:
3328       srcTy = TYPE_U32;
3329       /* fallthrough */
3330    case TGSI_OPCODE_CMP:
3331       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3332          src0 = fetchSrc(0, c);
3333          src1 = fetchSrc(1, c);
3334          src2 = fetchSrc(2, c);
3335          if (src1 == src2)
3336             mkMov(dst0[c], src1);
3337          else
3338             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
3339                   srcTy, dst0[c], srcTy, src1, src2, src0);
3340       }
3341       break;
3342    case TGSI_OPCODE_FRC:
3343       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3344          src0 = fetchSrc(0, c);
3345          val0 = getScratch();
3346          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
3347          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
3348       }
3349       break;
3350    case TGSI_OPCODE_ROUND:
3351       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3352          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
3353          ->rnd = ROUND_NI;
3354       break;
3355    case TGSI_OPCODE_SLT:
3356    case TGSI_OPCODE_SGE:
3357    case TGSI_OPCODE_SEQ:
3358    case TGSI_OPCODE_SGT:
3359    case TGSI_OPCODE_SLE:
3360    case TGSI_OPCODE_SNE:
3361    case TGSI_OPCODE_FSEQ:
3362    case TGSI_OPCODE_FSGE:
3363    case TGSI_OPCODE_FSLT:
3364    case TGSI_OPCODE_FSNE:
3365    case TGSI_OPCODE_ISGE:
3366    case TGSI_OPCODE_ISLT:
3367    case TGSI_OPCODE_USEQ:
3368    case TGSI_OPCODE_USGE:
3369    case TGSI_OPCODE_USLT:
3370    case TGSI_OPCODE_USNE:
3371       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3372          src0 = fetchSrc(0, c);
3373          src1 = fetchSrc(1, c);
3374          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3375       }
3376       break;
3377    case TGSI_OPCODE_VOTE_ALL:
3378    case TGSI_OPCODE_VOTE_ANY:
3379    case TGSI_OPCODE_VOTE_EQ:
3380       val0 = new_LValue(func, FILE_PREDICATE);
3381       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3382          mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, c), zero);
3383          mkOp1(op, dstTy, val0, val0)
3384             ->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3385          mkCvt(OP_CVT, TYPE_U32, dst0[c], TYPE_U8, val0);
3386       }
3387       break;
3388    case TGSI_OPCODE_BALLOT:
3389       if (!tgsi.getDst(0).isMasked(0)) {
3390          val0 = new_LValue(func, FILE_PREDICATE);
3391          mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, 0), zero);
3392          mkOp1(op, TYPE_U32, dst0[0], val0)->subOp = NV50_IR_SUBOP_VOTE_ANY;
3393       }
3394       if (!tgsi.getDst(0).isMasked(1))
3395          mkMov(dst0[1], zero, TYPE_U32);
3396       break;
3397    case TGSI_OPCODE_READ_FIRST:
3398       // ReadFirstInvocationARB(src) is implemented as
3399       // ReadInvocationARB(src, findLSB(ballot(true)))
3400       val0 = getScratch();
3401       mkOp1(OP_VOTE, TYPE_U32, val0, mkImm(1))->subOp = NV50_IR_SUBOP_VOTE_ANY;
3402       mkOp1(OP_BREV, TYPE_U32, val0, val0);
3403       mkOp1(OP_BFIND, TYPE_U32, val0, val0)->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3404       src1 = val0;
3405       /* fallthrough */
3406    case TGSI_OPCODE_READ_INVOC:
3407       if (tgsi.getOpcode() == TGSI_OPCODE_READ_INVOC)
3408          src1 = fetchSrc(1, 0);
3409       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3410          geni = mkOp3(op, dstTy, dst0[c], fetchSrc(0, c), src1, mkImm(0x1f));
3411          geni->subOp = NV50_IR_SUBOP_SHFL_IDX;
3412       }
3413       break;
3414    case TGSI_OPCODE_CLOCK:
3415       // Stick the 32-bit clock into the high dword of the logical result.
3416       if (!tgsi.getDst(0).isMasked(0))
3417          mkOp1(OP_MOV, TYPE_U32, dst0[0], zero);
3418       if (!tgsi.getDst(0).isMasked(1))
3419          mkOp1(OP_RDSV, TYPE_U32, dst0[1], mkSysVal(SV_CLOCK, 0))->fixed = 1;
3420       break;
3421    case TGSI_OPCODE_READ_HELPER:
3422       if (!tgsi.getDst(0).isMasked(0))
3423          mkOp1(OP_RDSV, TYPE_U32, dst0[0], mkSysVal(SV_THREAD_KILL, 0))
3424             ->fixed = 1;
3425       break;
3426    case TGSI_OPCODE_KILL_IF:
3427       val0 = new_LValue(func, FILE_PREDICATE);
3428       mask = 0;
3429       for (c = 0; c < 4; ++c) {
3430          const int s = tgsi.getSrc(0).getSwizzle(c);
3431          if (mask & (1 << s))
3432             continue;
3433          mask |= 1 << s;
3434          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
3435          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
3436       }
3437       break;
3438    case TGSI_OPCODE_KILL:
3439    case TGSI_OPCODE_DEMOTE:
3440       // TODO: Should we make KILL exit that invocation? Some old shaders
3441       // don't like that.
3442       mkOp(OP_DISCARD, TYPE_NONE, NULL);
3443       break;
3444    case TGSI_OPCODE_TEX:
3445    case TGSI_OPCODE_TEX_LZ:
3446    case TGSI_OPCODE_TXB:
3447    case TGSI_OPCODE_TXL:
3448    case TGSI_OPCODE_TXP:
3449    case TGSI_OPCODE_LODQ:
3450       //              R  S     L     C    Dx    Dy
3451       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
3452       break;
3453    case TGSI_OPCODE_TXD:
3454       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
3455       break;
3456    case TGSI_OPCODE_TG4:
3457       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
3458       break;
3459    case TGSI_OPCODE_TEX2:
3460       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
3461       break;
3462    case TGSI_OPCODE_TXB2:
3463    case TGSI_OPCODE_TXL2:
3464       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
3465       break;
3466    case TGSI_OPCODE_SAMPLE:
3467    case TGSI_OPCODE_SAMPLE_B:
3468    case TGSI_OPCODE_SAMPLE_D:
3469    case TGSI_OPCODE_SAMPLE_L:
3470    case TGSI_OPCODE_SAMPLE_C:
3471    case TGSI_OPCODE_SAMPLE_C_LZ:
3472       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
3473       break;
3474    case TGSI_OPCODE_TXF_LZ:
3475    case TGSI_OPCODE_TXF:
3476       handleTXF(dst0, 1, 0x03);
3477       break;
3478    case TGSI_OPCODE_SAMPLE_I:
3479       handleTXF(dst0, 1, 0x03);
3480       break;
3481    case TGSI_OPCODE_SAMPLE_I_MS:
3482       handleTXF(dst0, 1, 0x20);
3483       break;
3484    case TGSI_OPCODE_TXQ:
3485    case TGSI_OPCODE_SVIEWINFO:
3486       handleTXQ(dst0, TXQ_DIMS, 1);
3487       break;
3488    case TGSI_OPCODE_TXQS:
3489       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
3490       // be in .x
3491       dst0[1] = dst0[2] = dst0[3] = NULL;
3492       std::swap(dst0[0], dst0[2]);
3493       handleTXQ(dst0, TXQ_TYPE, 0);
3494       std::swap(dst0[0], dst0[2]);
3495       break;
3496    case TGSI_OPCODE_FBFETCH:
3497       handleFBFETCH(dst0);
3498       break;
3499    case TGSI_OPCODE_F2I:
3500    case TGSI_OPCODE_F2U:
3501       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3502          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
3503       break;
3504    case TGSI_OPCODE_I2F:
3505    case TGSI_OPCODE_U2F:
3506       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3507          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
3508       break;
3509    case TGSI_OPCODE_PK2H:
3510       val0 = getScratch();
3511       val1 = getScratch();
3512       mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
3513       mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
3514       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3515          mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
3516       break;
3517    case TGSI_OPCODE_UP2H:
3518       src0 = fetchSrc(0, 0);
3519       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3520          geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3521          geni->subOp = c & 1;
3522       }
3523       break;
3524    case TGSI_OPCODE_EMIT:
3525       /* export the saved viewport index */
3526       if (viewport != NULL) {
3527          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3528                                   info_out->out[info->io.viewportId].slot[0] * 4);
3529          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3530       }
3531       /* handle user clip planes for each emitted vertex */
3532       if (info_out->io.genUserClip > 0)
3533          handleUserClipPlanes();
3534       /* fallthrough */
3535    case TGSI_OPCODE_ENDPRIM:
3536    {
3537       // get vertex stream (must be immediate)
3538       unsigned int stream = tgsi.getSrc(0).getValueU32(0, code->immd.data);
3539       if (stream && op == OP_RESTART)
3540          break;
3541       if (info_out->prop.gp.maxVertices == 0)
3542          break;
3543       src0 = mkImm(stream);
3544       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3545       break;
3546    }
3547    case TGSI_OPCODE_IF:
3548    case TGSI_OPCODE_UIF:
3549    {
3550       BasicBlock *ifBB = new BasicBlock(func);
3551 
3552       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3553       condBBs.push(bb);
3554       joinBBs.push(bb);
3555 
3556       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3557 
3558       setPosition(ifBB, true);
3559    }
3560       break;
3561    case TGSI_OPCODE_ELSE:
3562    {
3563       BasicBlock *elseBB = new BasicBlock(func);
3564       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3565 
3566       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3567       condBBs.push(bb);
3568 
3569       forkBB->getExit()->asFlow()->target.bb = elseBB;
3570       if (!bb->isTerminated())
3571          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3572 
3573       setPosition(elseBB, true);
3574    }
3575       break;
3576    case TGSI_OPCODE_ENDIF:
3577    {
3578       BasicBlock *convBB = new BasicBlock(func);
3579       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3580       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3581 
3582       if (!bb->isTerminated()) {
3583          // we only want join if none of the clauses ended with CONT/BREAK/RET
3584          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3585             insertConvergenceOps(convBB, forkBB);
3586          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3587          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3588       }
3589 
3590       if (prevBB->getExit()->op == OP_BRA) {
3591          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3592          prevBB->getExit()->asFlow()->target.bb = convBB;
3593       }
3594       setPosition(convBB, true);
3595    }
3596       break;
3597    case TGSI_OPCODE_BGNLOOP:
3598    {
3599       BasicBlock *lbgnBB = new BasicBlock(func);
3600       BasicBlock *lbrkBB = new BasicBlock(func);
3601 
3602       loopBBs.push(lbgnBB);
3603       breakBBs.push(lbrkBB);
3604       if (loopBBs.getSize() > func->loopNestingBound)
3605          func->loopNestingBound++;
3606 
3607       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3608 
3609       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3610       setPosition(lbgnBB, true);
3611       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3612    }
3613       break;
3614    case TGSI_OPCODE_ENDLOOP:
3615    {
3616       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3617 
3618       if (!bb->isTerminated()) {
3619          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3620          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3621       }
3622       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3623 
3624       // If the loop never breaks (e.g. only has RET's inside), then there
3625       // will be no way to get to the break bb. However BGNLOOP will have
3626       // already made a PREBREAK to it, so it must be in the CFG.
3627       if (getBB()->cfg.incidentCount() == 0)
3628          loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3629    }
3630       break;
3631    case TGSI_OPCODE_BRK:
3632    {
3633       if (bb->isTerminated())
3634          break;
3635       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3636       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3637       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3638    }
3639       break;
3640    case TGSI_OPCODE_CONT:
3641    {
3642       if (bb->isTerminated())
3643          break;
3644       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3645       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3646       contBB->explicitCont = true;
3647       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3648    }
3649       break;
3650    case TGSI_OPCODE_BGNSUB:
3651    {
3652       Subroutine *s = getSubroutine(ip);
3653       BasicBlock *entry = new BasicBlock(s->f);
3654       BasicBlock *leave = new BasicBlock(s->f);
3655 
3656       // multiple entrypoints possible, keep the graph connected
3657       if (prog->getType() == Program::TYPE_COMPUTE)
3658          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3659 
3660       sub.cur = s;
3661       s->f->setEntry(entry);
3662       s->f->setExit(leave);
3663       setPosition(entry, true);
3664       return true;
3665    }
3666    case TGSI_OPCODE_ENDSUB:
3667    {
3668       sub.cur = getSubroutine(prog->main);
3669       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3670       return true;
3671    }
3672    case TGSI_OPCODE_CAL:
3673    {
3674       Subroutine *s = getSubroutine(tgsi.getLabel());
3675       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3676       func->call.attach(&s->f->call, Graph::Edge::TREE);
3677       return true;
3678    }
3679    case TGSI_OPCODE_RET:
3680    {
3681       if (bb->isTerminated())
3682          return true;
3683       BasicBlock *leave = BasicBlock::get(func->cfgExit);
3684 
3685       if (!isEndOfSubroutine(ip + 1)) {
3686          // insert a PRERET at the entry if this is an early return
3687          // (only needed for sharing code in the epilogue)
3688          BasicBlock *root = BasicBlock::get(func->cfg.getRoot());
3689          if (root->getEntry() == NULL || root->getEntry()->op != OP_PRERET) {
3690             BasicBlock *pos = getBB();
3691             setPosition(root, false);
3692             mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3693             setPosition(pos, true);
3694          }
3695       }
3696       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3697       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3698    }
3699       break;
3700    case TGSI_OPCODE_END:
3701    {
3702       // attach and generate epilogue code
3703       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3704       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3705       setPosition(epilogue, true);
3706       if (prog->getType() == Program::TYPE_FRAGMENT)
3707          exportOutputs();
3708       if ((prog->getType() == Program::TYPE_VERTEX ||
3709            prog->getType() == Program::TYPE_TESSELLATION_EVAL
3710           ) && info_out->io.genUserClip > 0)
3711          handleUserClipPlanes();
3712       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3713    }
3714       break;
3715    case TGSI_OPCODE_SWITCH:
3716    case TGSI_OPCODE_CASE:
3717       ERROR("switch/case opcode encountered, should have been lowered\n");
3718       abort();
3719       break;
3720    case TGSI_OPCODE_LOAD:
3721       handleLOAD(dst0);
3722       break;
3723    case TGSI_OPCODE_STORE:
3724       handleSTORE();
3725       break;
3726    case TGSI_OPCODE_BARRIER:
3727       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3728       geni->fixed = 1;
3729       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3730       break;
3731    case TGSI_OPCODE_MEMBAR:
3732    {
3733       uint32_t level = tgsi.getSrc(0).getValueU32(0, code->immd.data);
3734       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3735       geni->fixed = 1;
3736       if (!(level & ~(TGSI_MEMBAR_THREAD_GROUP | TGSI_MEMBAR_SHARED)))
3737          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3738       else
3739          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3740    }
3741       break;
3742    case TGSI_OPCODE_ATOMUADD:
3743    case TGSI_OPCODE_ATOMXCHG:
3744    case TGSI_OPCODE_ATOMCAS:
3745    case TGSI_OPCODE_ATOMAND:
3746    case TGSI_OPCODE_ATOMOR:
3747    case TGSI_OPCODE_ATOMXOR:
3748    case TGSI_OPCODE_ATOMUMIN:
3749    case TGSI_OPCODE_ATOMIMIN:
3750    case TGSI_OPCODE_ATOMUMAX:
3751    case TGSI_OPCODE_ATOMIMAX:
3752    case TGSI_OPCODE_ATOMFADD:
3753    case TGSI_OPCODE_ATOMDEC_WRAP:
3754    case TGSI_OPCODE_ATOMINC_WRAP:
3755       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3756       break;
3757    case TGSI_OPCODE_RESQ:
3758       if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER) {
3759          Value *ind = NULL;
3760          if (tgsi.getSrc(0).isIndirect(0))
3761             ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
3762          geni = mkOp1(OP_BUFQ, TYPE_U32, dst0[0],
3763                       makeSym(tgsi.getSrc(0).getFile(),
3764                               tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3765          if (ind)
3766             geni->setIndirect(0, 1, ind);
3767       } else {
3768          TexInstruction *texi = new_TexInstruction(func, OP_SUQ);
3769          for (int c = 0, d = 0; c < 4; ++c) {
3770             if (dst0[c]) {
3771                texi->setDef(d++, dst0[c]);
3772                texi->tex.mask |= 1 << c;
3773             }
3774          }
3775          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMAGE) {
3776             texi->tex.r = tgsi.getSrc(0).getIndex(0);
3777             if (tgsi.getSrc(0).isIndirect(0))
3778                texi->setIndirectR(fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, NULL));
3779          } else {
3780             texi->tex.bindless = true;
3781             texi->setIndirectR(fetchSrc(0, 0));
3782          }
3783          texi->tex.target = tgsi.getImageTarget();
3784 
3785          bb->insertTail(texi);
3786       }
3787       break;
3788    case TGSI_OPCODE_IBFE:
3789    case TGSI_OPCODE_UBFE:
3790       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3791          src0 = fetchSrc(0, c);
3792          val0 = getScratch();
3793          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3794              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3795             loadImm(val0, (tgsi.getSrc(2).getValueU32(c, code->immd.data) << 8) |
3796                     tgsi.getSrc(1).getValueU32(c, code->immd.data));
3797          } else {
3798             src1 = fetchSrc(1, c);
3799             src2 = fetchSrc(2, c);
3800             mkOp3(OP_INSBF, TYPE_U32, val0, src2, mkImm(0x808), src1);
3801          }
3802          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, val0);
3803       }
3804       break;
3805    case TGSI_OPCODE_BFI:
3806       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3807          src0 = fetchSrc(0, c);
3808          src1 = fetchSrc(1, c);
3809          src2 = fetchSrc(2, c);
3810          src3 = fetchSrc(3, c);
3811          val0 = getScratch();
3812          mkOp3(OP_INSBF, TYPE_U32, val0, src3, mkImm(0x808), src2);
3813          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, val0, src0);
3814       }
3815       break;
3816    case TGSI_OPCODE_LSB:
3817       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3818          src0 = fetchSrc(0, c);
3819          val0 = getScratch();
3820          mkOp1(OP_BREV, TYPE_U32, val0, src0);
3821          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], val0);
3822          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3823       }
3824       break;
3825    case TGSI_OPCODE_IMSB:
3826    case TGSI_OPCODE_UMSB:
3827       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3828          src0 = fetchSrc(0, c);
3829          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3830       }
3831       break;
3832    case TGSI_OPCODE_BREV:
3833       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3834          src0 = fetchSrc(0, c);
3835          mkOp1(OP_BREV, TYPE_U32, dst0[c], src0);
3836       }
3837       break;
3838    case TGSI_OPCODE_POPC:
3839       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3840          src0 = fetchSrc(0, c);
3841          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3842       }
3843       break;
3844    case TGSI_OPCODE_INTERP_CENTROID:
3845    case TGSI_OPCODE_INTERP_SAMPLE:
3846    case TGSI_OPCODE_INTERP_OFFSET:
3847       handleINTERP(dst0);
3848       break;
3849    case TGSI_OPCODE_I642F:
3850    case TGSI_OPCODE_U642F:
3851    case TGSI_OPCODE_D2I:
3852    case TGSI_OPCODE_D2U:
3853    case TGSI_OPCODE_D2F: {
3854       int pos = 0;
3855       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3856          Value *dreg = getSSA(8);
3857          src0 = fetchSrc(0, pos);
3858          src1 = fetchSrc(0, pos + 1);
3859          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3860          Instruction *cvt = mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3861          if (!isFloatType(dstTy))
3862             cvt->rnd = ROUND_Z;
3863          pos += 2;
3864       }
3865       break;
3866    }
3867    case TGSI_OPCODE_I2I64:
3868       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3869          dst0[c] = fetchSrc(0, c / 2);
3870          mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(NULL, 31));
3871          c++;
3872       }
3873       break;
3874    case TGSI_OPCODE_U2I64:
3875       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3876          dst0[c] = fetchSrc(0, c / 2);
3877          dst0[c + 1] = zero;
3878          c++;
3879       }
3880       break;
3881    case TGSI_OPCODE_F2I64:
3882    case TGSI_OPCODE_F2U64:
3883    case TGSI_OPCODE_I2D:
3884    case TGSI_OPCODE_U2D:
3885    case TGSI_OPCODE_F2D:
3886       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3887          Value *dreg = getSSA(8);
3888          Instruction *cvt = mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3889          if (!isFloatType(dstTy))
3890             cvt->rnd = ROUND_Z;
3891          mkSplit(&dst0[c], 4, dreg);
3892          c++;
3893       }
3894       break;
3895    case TGSI_OPCODE_D2I64:
3896    case TGSI_OPCODE_D2U64:
3897    case TGSI_OPCODE_I642D:
3898    case TGSI_OPCODE_U642D:
3899       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3900          src0 = getSSA(8);
3901          Value *dst = getSSA(8), *tmp[2];
3902          tmp[0] = fetchSrc(0, c);
3903          tmp[1] = fetchSrc(0, c + 1);
3904          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3905          Instruction *cvt = mkCvt(OP_CVT, dstTy, dst, srcTy, src0);
3906          if (!isFloatType(dstTy))
3907             cvt->rnd = ROUND_Z;
3908          mkSplit(&dst0[c], 4, dst);
3909          c++;
3910       }
3911       break;
3912    case TGSI_OPCODE_I64NEG:
3913       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3914          src0 = getSSA(8);
3915          Value *dst = getSSA(8), *tmp[2];
3916          tmp[0] = fetchSrc(0, c);
3917          tmp[1] = fetchSrc(0, c + 1);
3918          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3919          mkOp2(OP_SUB, dstTy, dst, zero, src0);
3920          mkSplit(&dst0[c], 4, dst);
3921          c++;
3922       }
3923       break;
3924    case TGSI_OPCODE_I64ABS:
3925       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3926          src0 = getSSA(8);
3927          Value *neg = getSSA(8), *srcComp[2], *negComp[2];
3928          srcComp[0] = fetchSrc(0, c);
3929          srcComp[1] = fetchSrc(0, c + 1);
3930          mkOp2(OP_MERGE, TYPE_U64, src0, srcComp[0], srcComp[1]);
3931          mkOp2(OP_SUB, dstTy, neg, zero, src0);
3932          mkSplit(negComp, 4, neg);
3933          mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c], TYPE_S32,
3934                negComp[0], srcComp[0], srcComp[1]);
3935          mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c + 1], TYPE_S32,
3936                negComp[1], srcComp[1], srcComp[1]);
3937          c++;
3938       }
3939       break;
3940    case TGSI_OPCODE_DABS:
3941    case TGSI_OPCODE_DNEG:
3942    case TGSI_OPCODE_DRCP:
3943    case TGSI_OPCODE_DSQRT:
3944    case TGSI_OPCODE_DRSQ:
3945    case TGSI_OPCODE_DTRUNC:
3946    case TGSI_OPCODE_DCEIL:
3947    case TGSI_OPCODE_DFLR:
3948       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3949          src0 = getSSA(8);
3950          Value *dst = getSSA(8), *tmp[2];
3951          tmp[0] = fetchSrc(0, c);
3952          tmp[1] = fetchSrc(0, c + 1);
3953          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3954          mkOp1(op, dstTy, dst, src0);
3955          mkSplit(&dst0[c], 4, dst);
3956          c++;
3957       }
3958       break;
3959    case TGSI_OPCODE_DFRAC:
3960       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3961          src0 = getSSA(8);
3962          Value *dst = getSSA(8), *tmp[2];
3963          tmp[0] = fetchSrc(0, c);
3964          tmp[1] = fetchSrc(0, c + 1);
3965          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3966          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3967          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3968          mkSplit(&dst0[c], 4, dst);
3969          c++;
3970       }
3971       break;
3972    case TGSI_OPCODE_U64SEQ:
3973    case TGSI_OPCODE_U64SNE:
3974    case TGSI_OPCODE_U64SLT:
3975    case TGSI_OPCODE_U64SGE:
3976    case TGSI_OPCODE_I64SLT:
3977    case TGSI_OPCODE_I64SGE:
3978    case TGSI_OPCODE_DSLT:
3979    case TGSI_OPCODE_DSGE:
3980    case TGSI_OPCODE_DSEQ:
3981    case TGSI_OPCODE_DSNE: {
3982       int pos = 0;
3983       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3984          Value *tmp[2];
3985 
3986          src0 = getSSA(8);
3987          src1 = getSSA(8);
3988          tmp[0] = fetchSrc(0, pos);
3989          tmp[1] = fetchSrc(0, pos + 1);
3990          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3991          tmp[0] = fetchSrc(1, pos);
3992          tmp[1] = fetchSrc(1, pos + 1);
3993          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3994          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3995          pos += 2;
3996       }
3997       break;
3998    }
3999    case TGSI_OPCODE_U64MIN:
4000    case TGSI_OPCODE_U64MAX:
4001    case TGSI_OPCODE_I64MIN:
4002    case TGSI_OPCODE_I64MAX: {
4003       dstTy = isSignedIntType(dstTy) ? TYPE_S32 : TYPE_U32;
4004       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4005          Value *flag = getSSA(1, FILE_FLAGS);
4006          src0 = fetchSrc(0, c + 1);
4007          src1 = fetchSrc(1, c + 1);
4008          geni = mkOp2(op, dstTy, dst0[c + 1], src0, src1);
4009          geni->subOp = NV50_IR_SUBOP_MINMAX_HIGH;
4010          geni->setFlagsDef(1, flag);
4011 
4012          src0 = fetchSrc(0, c);
4013          src1 = fetchSrc(1, c);
4014          geni = mkOp2(op, TYPE_U32, dst0[c], src0, src1);
4015          geni->subOp = NV50_IR_SUBOP_MINMAX_LOW;
4016          geni->setFlagsSrc(2, flag);
4017 
4018          c++;
4019       }
4020       break;
4021    }
4022    case TGSI_OPCODE_U64SHL:
4023    case TGSI_OPCODE_I64SHR:
4024    case TGSI_OPCODE_U64SHR:
4025       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4026          src0 = getSSA(8);
4027          Value *dst = getSSA(8), *tmp[2];
4028          tmp[0] = fetchSrc(0, c);
4029          tmp[1] = fetchSrc(0, c + 1);
4030          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4031          // Theoretically src1 is a 64-bit value but in practice only the low
4032          // bits matter. The IR expects this to be a 32-bit value.
4033          src1 = fetchSrc(1, c);
4034          mkOp2(op, dstTy, dst, src0, src1);
4035          mkSplit(&dst0[c], 4, dst);
4036          c++;
4037       }
4038       break;
4039    case TGSI_OPCODE_U64ADD:
4040    case TGSI_OPCODE_U64MUL:
4041    case TGSI_OPCODE_DADD:
4042    case TGSI_OPCODE_DMUL:
4043    case TGSI_OPCODE_DDIV:
4044    case TGSI_OPCODE_DMAX:
4045    case TGSI_OPCODE_DMIN:
4046       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4047          src0 = getSSA(8);
4048          src1 = getSSA(8);
4049          Value *dst = getSSA(8), *tmp[2];
4050          tmp[0] = fetchSrc(0, c);
4051          tmp[1] = fetchSrc(0, c + 1);
4052          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4053          tmp[0] = fetchSrc(1, c);
4054          tmp[1] = fetchSrc(1, c + 1);
4055          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4056          mkOp2(op, dstTy, dst, src0, src1);
4057          mkSplit(&dst0[c], 4, dst);
4058          c++;
4059       }
4060       break;
4061    case TGSI_OPCODE_DMAD:
4062    case TGSI_OPCODE_DFMA:
4063       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4064          src0 = getSSA(8);
4065          src1 = getSSA(8);
4066          src2 = getSSA(8);
4067          Value *dst = getSSA(8), *tmp[2];
4068          tmp[0] = fetchSrc(0, c);
4069          tmp[1] = fetchSrc(0, c + 1);
4070          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4071          tmp[0] = fetchSrc(1, c);
4072          tmp[1] = fetchSrc(1, c + 1);
4073          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4074          tmp[0] = fetchSrc(2, c);
4075          tmp[1] = fetchSrc(2, c + 1);
4076          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
4077          mkOp3(op, dstTy, dst, src0, src1, src2);
4078          mkSplit(&dst0[c], 4, dst);
4079          c++;
4080       }
4081       break;
4082    case TGSI_OPCODE_DROUND:
4083       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4084          src0 = getSSA(8);
4085          Value *dst = getSSA(8), *tmp[2];
4086          tmp[0] = fetchSrc(0, c);
4087          tmp[1] = fetchSrc(0, c + 1);
4088          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4089          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
4090          ->rnd = ROUND_NI;
4091          mkSplit(&dst0[c], 4, dst);
4092          c++;
4093       }
4094       break;
4095    case TGSI_OPCODE_DSSG:
4096       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4097          src0 = getSSA(8);
4098          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
4099          tmp[0] = fetchSrc(0, c);
4100          tmp[1] = fetchSrc(0, c + 1);
4101          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4102 
4103          val0 = getScratch();
4104          val1 = getScratch();
4105          // The zero is wrong here since it's only 32-bit, but it works out in
4106          // the end since it gets replaced with $r63.
4107          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
4108          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
4109          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
4110          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
4111          mkSplit(&dst0[c], 4, dst);
4112          c++;
4113       }
4114       break;
4115    case TGSI_OPCODE_I64SSG:
4116       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4117          src0 = getSSA(8);
4118          Value *tmp[2];
4119          tmp[0] = fetchSrc(0, c);
4120          tmp[1] = fetchSrc(0, c + 1);
4121          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4122 
4123          val0 = getScratch();
4124          val1 = getScratch();
4125          mkCmp(OP_SET, CC_GT, TYPE_U32, val0, TYPE_S64, src0, zero);
4126          mkCmp(OP_SET, CC_LT, TYPE_U32, val1, TYPE_S64, src0, zero);
4127          mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
4128          mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(0, 31));
4129          c++;
4130       }
4131       break;
4132    default:
4133       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
4134       assert(0);
4135       break;
4136    }
4137 
4138    if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
4139       for (c = 0; c < 4; ++c) {
4140          if (!dst0[c])
4141             continue;
4142          if (dst0[c] != rDst0[c])
4143             mkMov(rDst0[c], dst0[c]);
4144          storeDst(0, c, rDst0[c]);
4145       }
4146    }
4147    vtxBaseValid = 0;
4148 
4149    return true;
4150 }
4151 
4152 void
exportOutputs()4153 Converter::exportOutputs()
4154 {
4155    if (info->io.alphaRefBase) {
4156       for (unsigned int i = 0; i < info_out->numOutputs; ++i) {
4157          if (info_out->out[i].sn != TGSI_SEMANTIC_COLOR ||
4158              info_out->out[i].si != 0)
4159             continue;
4160          const unsigned int c = 3;
4161          if (!oData.exists(sub.cur->values, i, c))
4162             continue;
4163          Value *val = oData.load(sub.cur->values, i, c, NULL);
4164          if (!val)
4165             continue;
4166 
4167          Symbol *ref = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
4168                                 TYPE_U32, info->io.alphaRefBase);
4169          Value *pred = new_LValue(func, FILE_PREDICATE);
4170          mkCmp(OP_SET, CC_TR, TYPE_U32, pred, TYPE_F32, val,
4171                mkLoadv(TYPE_U32, ref, NULL))
4172             ->subOp = 1;
4173          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_NOT_P, pred);
4174       }
4175    }
4176 
4177    for (unsigned int i = 0; i < info_out->numOutputs; ++i) {
4178       for (unsigned int c = 0; c < 4; ++c) {
4179          if (!oData.exists(sub.cur->values, i, c))
4180             continue;
4181          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
4182                                 info_out->out[i].slot[c] * 4);
4183          Value *val = oData.load(sub.cur->values, i, c, NULL);
4184          if (val) {
4185             if (info_out->out[i].sn == TGSI_SEMANTIC_POSITION)
4186                mkOp1(OP_SAT, TYPE_F32, val, val);
4187             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
4188          }
4189       }
4190    }
4191 }
4192 
Converter(Program * ir,const tgsi::Source * code,nv50_ir_prog_info_out * info_out)4193 Converter::Converter(Program *ir, const tgsi::Source *code, nv50_ir_prog_info_out *info_out)
4194 :    ConverterCommon(ir, code->info, info_out),
4195      code(code),
4196      tgsi(NULL),
4197      tData(this), lData(this), aData(this), oData(this)
4198 {
4199    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
4200    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
4201    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
4202 
4203    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
4204    lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
4205    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
4206    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
4207 
4208    zero = mkImm((uint32_t)0);
4209 
4210    vtxBaseValid = 0;
4211 }
4212 
~Converter()4213 Converter::~Converter()
4214 {
4215 }
4216 
4217 inline const Converter::Location *
getValueLocation(Subroutine * s,Value * v)4218 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
4219 {
4220    ValueMap::l_iterator it = s->values.l.find(v);
4221    return it == s->values.l.end() ? NULL : &it->second;
4222 }
4223 
4224 template<typename T> inline void
updateCallArgs(Instruction * i,void (Instruction::* setArg)(int,Value *),T (Function::* proto))4225 Converter::BindArgumentsPass::updateCallArgs(
4226    Instruction *i, void (Instruction::*setArg)(int, Value *),
4227    T (Function::*proto))
4228 {
4229    Function *g = i->asFlow()->target.fn;
4230    Subroutine *subg = conv.getSubroutine(g);
4231 
4232    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
4233       Value *v = (g->*proto)[a].get();
4234       const Converter::Location &l = *getValueLocation(subg, v);
4235       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
4236 
4237       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
4238    }
4239 }
4240 
4241 template<typename T> inline void
updatePrototype(BitSet * set,void (Function::* updateSet)(),T (Function::* proto))4242 Converter::BindArgumentsPass::updatePrototype(
4243    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
4244 {
4245    (func->*updateSet)();
4246 
4247    for (unsigned i = 0; i < set->getSize(); ++i) {
4248       Value *v = func->getLValue(i);
4249       const Converter::Location *l = getValueLocation(sub, v);
4250 
4251       // only include values with a matching TGSI register
4252       if (set->test(i) && l && !conv.code->locals.count(*l))
4253          (func->*proto).push_back(v);
4254    }
4255 }
4256 
4257 bool
visit(Function * f)4258 Converter::BindArgumentsPass::visit(Function *f)
4259 {
4260    sub = conv.getSubroutine(f);
4261 
4262    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
4263         !bi.end(); bi.next()) {
4264       for (Instruction *i = BasicBlock::get(bi)->getFirst();
4265            i; i = i->next) {
4266          if (i->op == OP_CALL && !i->asFlow()->builtin) {
4267             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
4268             updateCallArgs(i, &Instruction::setDef, &Function::outs);
4269          }
4270       }
4271    }
4272 
4273    if (func == prog->main /* && prog->getType() != Program::TYPE_COMPUTE */)
4274       return true;
4275    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
4276                    &Function::buildLiveSets, &Function::ins);
4277    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
4278                    &Function::buildDefSets, &Function::outs);
4279 
4280    return true;
4281 }
4282 
4283 bool
run()4284 Converter::run()
4285 {
4286    BasicBlock *entry = new BasicBlock(prog->main);
4287    BasicBlock *leave = new BasicBlock(prog->main);
4288 
4289    prog->main->setEntry(entry);
4290    prog->main->setExit(leave);
4291 
4292    setPosition(entry, true);
4293    sub.cur = getSubroutine(prog->main);
4294 
4295    if (info_out->io.genUserClip > 0) {
4296       for (int c = 0; c < 4; ++c)
4297          clipVtx[c] = getScratch();
4298    }
4299 
4300    switch (prog->getType()) {
4301    case Program::TYPE_TESSELLATION_CONTROL:
4302       outBase = mkOp2v(
4303          OP_SUB, TYPE_U32, getSSA(),
4304          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
4305          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
4306       break;
4307    case Program::TYPE_FRAGMENT: {
4308       Symbol *sv = mkSysVal(SV_POSITION, 3);
4309       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
4310       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
4311       break;
4312    }
4313    default:
4314       break;
4315    }
4316 
4317    if (info->io.viewportId >= 0)
4318       viewport = getScratch();
4319    else
4320       viewport = NULL;
4321 
4322    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
4323       if (!handleInstruction(&code->insns[ip]))
4324          return false;
4325    }
4326 
4327    if (!BindArgumentsPass(*this).run(prog))
4328       return false;
4329 
4330    return true;
4331 }
4332 
4333 } // unnamed namespace
4334 
4335 namespace nv50_ir {
4336 
4337 bool
makeFromTGSI(struct nv50_ir_prog_info * info,struct nv50_ir_prog_info_out * info_out)4338 Program::makeFromTGSI(struct nv50_ir_prog_info *info,
4339                       struct nv50_ir_prog_info_out *info_out)
4340 {
4341    tgsi::Source src(info, info_out, this);
4342    if (!src.scanSource())
4343       return false;
4344    tlsSize = info_out->bin.tlsSpace;
4345 
4346    Converter builder(this, &src, info_out);
4347    return builder.run();
4348 }
4349 
4350 } // namespace nv50_ir
4351