1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "nv50_ir.h"
24 #include "nv50_ir_build_util.h"
25
26 namespace nv50_ir {
27
BuildUtil()28 BuildUtil::BuildUtil()
29 {
30 init(NULL);
31 }
32
BuildUtil(Program * prog)33 BuildUtil::BuildUtil(Program *prog)
34 {
35 init(prog);
36 }
37
38 void
init(Program * prog)39 BuildUtil::init(Program *prog)
40 {
41 this->prog = prog;
42
43 func = NULL;
44 bb = NULL;
45 pos = NULL;
46
47 tail = false;
48
49 memset(imms, 0, sizeof(imms));
50 immCount = 0;
51 }
52
53 void
addImmediate(ImmediateValue * imm)54 BuildUtil::addImmediate(ImmediateValue *imm)
55 {
56 if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
57 return;
58
59 unsigned int pos = u32Hash(imm->reg.data.u32);
60
61 while (imms[pos])
62 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
63 imms[pos] = imm;
64 immCount++;
65 }
66
67 Instruction *
mkOp1(operation op,DataType ty,Value * dst,Value * src)68 BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
69 {
70 Instruction *insn = new_Instruction(func, op, ty);
71
72 insn->setDef(0, dst);
73 insn->setSrc(0, src);
74
75 insert(insn);
76 return insn;
77 }
78
79 Instruction *
mkOp2(operation op,DataType ty,Value * dst,Value * src0,Value * src1)80 BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
81 Value *src0, Value *src1)
82 {
83 Instruction *insn = new_Instruction(func, op, ty);
84
85 insn->setDef(0, dst);
86 insn->setSrc(0, src0);
87 insn->setSrc(1, src1);
88
89 insert(insn);
90 return insn;
91 }
92
93 Instruction *
mkOp3(operation op,DataType ty,Value * dst,Value * src0,Value * src1,Value * src2)94 BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
95 Value *src0, Value *src1, Value *src2)
96 {
97 Instruction *insn = new_Instruction(func, op, ty);
98
99 insn->setDef(0, dst);
100 insn->setSrc(0, src0);
101 insn->setSrc(1, src1);
102 insn->setSrc(2, src2);
103
104 insert(insn);
105 return insn;
106 }
107
108 Instruction *
mkLoad(DataType ty,Value * dst,Symbol * mem,Value * ptr)109 BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
110 {
111 Instruction *insn = new_Instruction(func, OP_LOAD, ty);
112
113 insn->setDef(0, dst);
114 insn->setSrc(0, mem);
115 if (ptr)
116 insn->setIndirect(0, 0, ptr);
117
118 insert(insn);
119 return insn;
120 }
121
122 Instruction *
mkStore(operation op,DataType ty,Symbol * mem,Value * ptr,Value * stVal)123 BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
124 Value *stVal)
125 {
126 Instruction *insn = new_Instruction(func, op, ty);
127
128 insn->setSrc(0, mem);
129 insn->setSrc(1, stVal);
130 if (ptr)
131 insn->setIndirect(0, 0, ptr);
132
133 insert(insn);
134 return insn;
135 }
136
137 Instruction *
mkFetch(Value * dst,DataType ty,DataFile file,int32_t offset,Value * attrRel,Value * primRel)138 BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
139 Value *attrRel, Value *primRel)
140 {
141 Symbol *sym = mkSymbol(file, 0, ty, offset);
142
143 Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
144
145 insn->setIndirect(0, 0, attrRel);
146 insn->setIndirect(0, 1, primRel);
147
148 // already inserted
149 return insn;
150 }
151
152 Instruction *
mkInterp(unsigned mode,Value * dst,int32_t offset,Value * rel)153 BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
154 {
155 operation op = OP_LINTERP;
156 DataType ty = TYPE_F32;
157
158 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
159 ty = TYPE_U32;
160 else
161 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
162 op = OP_PINTERP;
163
164 Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
165
166 Instruction *insn = mkOp1(op, ty, dst, sym);
167 insn->setIndirect(0, 0, rel);
168 insn->setInterpolate(mode);
169 return insn;
170 }
171
172 Instruction *
mkMov(Value * dst,Value * src,DataType ty)173 BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
174 {
175 Instruction *insn = new_Instruction(func, OP_MOV, ty);
176
177 insn->setDef(0, dst);
178 insn->setSrc(0, src);
179
180 insert(insn);
181 return insn;
182 }
183
184 Instruction *
mkMovToReg(int id,Value * src)185 BuildUtil::mkMovToReg(int id, Value *src)
186 {
187 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
188
189 insn->setDef(0, new_LValue(func, FILE_GPR));
190 insn->getDef(0)->reg.data.id = id;
191 insn->setSrc(0, src);
192
193 insert(insn);
194 return insn;
195 }
196
197 Instruction *
mkMovFromReg(Value * dst,int id)198 BuildUtil::mkMovFromReg(Value *dst, int id)
199 {
200 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
201
202 insn->setDef(0, dst);
203 insn->setSrc(0, new_LValue(func, FILE_GPR));
204 insn->getSrc(0)->reg.data.id = id;
205
206 insert(insn);
207 return insn;
208 }
209
210 Instruction *
mkCvt(operation op,DataType dstTy,Value * dst,DataType srcTy,Value * src)211 BuildUtil::mkCvt(operation op,
212 DataType dstTy, Value *dst, DataType srcTy, Value *src)
213 {
214 Instruction *insn = new_Instruction(func, op, dstTy);
215
216 insn->setType(dstTy, srcTy);
217 insn->setDef(0, dst);
218 insn->setSrc(0, src);
219
220 insert(insn);
221 return insn;
222 }
223
224 CmpInstruction *
mkCmp(operation op,CondCode cc,DataType dstTy,Value * dst,DataType srcTy,Value * src0,Value * src1,Value * src2)225 BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst,
226 DataType srcTy, Value *src0, Value *src1, Value *src2)
227 {
228 CmpInstruction *insn = new_CmpInstruction(func, op);
229
230 insn->setType((dst->reg.file == FILE_PREDICATE ||
231 dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy);
232 insn->setCondition(cc);
233 insn->setDef(0, dst);
234 insn->setSrc(0, src0);
235 insn->setSrc(1, src1);
236 if (src2)
237 insn->setSrc(2, src2);
238
239 if (dst->reg.file == FILE_FLAGS)
240 insn->flagsDef = 0;
241
242 insert(insn);
243 return insn;
244 }
245
246 TexInstruction *
mkTex(operation op,TexTarget targ,uint16_t tic,uint16_t tsc,const std::vector<Value * > & def,const std::vector<Value * > & src)247 BuildUtil::mkTex(operation op, TexTarget targ,
248 uint16_t tic, uint16_t tsc,
249 const std::vector<Value *> &def,
250 const std::vector<Value *> &src)
251 {
252 TexInstruction *tex = new_TexInstruction(func, op);
253
254 for (size_t d = 0; d < def.size() && def[d]; ++d)
255 tex->setDef(d, def[d]);
256 for (size_t s = 0; s < src.size() && src[s]; ++s)
257 tex->setSrc(s, src[s]);
258
259 tex->setTexture(targ, tic, tsc);
260
261 insert(tex);
262 return tex;
263 }
264
265 Instruction *
mkQuadop(uint8_t q,Value * def,uint8_t l,Value * src0,Value * src1)266 BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
267 {
268 Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
269 quadop->subOp = q;
270 quadop->lanes = l;
271 return quadop;
272 }
273
274 Instruction *
mkSelect(Value * pred,Value * dst,Value * trSrc,Value * flSrc)275 BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
276 {
277 LValue *def0 = getSSA();
278 LValue *def1 = getSSA();
279
280 mkMov(def0, trSrc)->setPredicate(CC_P, pred);
281 mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
282
283 return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
284 }
285
286 Instruction *
mkSplit(Value * h[2],uint8_t halfSize,Value * val)287 BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
288 {
289 Instruction *insn = NULL;
290
291 const DataType fTy = typeOfSize(halfSize * 2);
292
293 if (val->reg.file == FILE_IMMEDIATE)
294 val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
295
296 if (isMemoryFile(val->reg.file)) {
297 h[0] = cloneShallow(getFunction(), val);
298 h[1] = cloneShallow(getFunction(), val);
299 h[0]->reg.size = halfSize;
300 h[1]->reg.size = halfSize;
301 h[1]->reg.data.offset += halfSize;
302 } else {
303 h[0] = getSSA(halfSize, val->reg.file);
304 h[1] = getSSA(halfSize, val->reg.file);
305 insn = mkOp1(OP_SPLIT, fTy, h[0], val);
306 insn->setDef(1, h[1]);
307 }
308 return insn;
309 }
310
311 FlowInstruction *
mkFlow(operation op,void * targ,CondCode cc,Value * pred)312 BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
313 {
314 FlowInstruction *insn = new_FlowInstruction(func, op, targ);
315
316 if (pred)
317 insn->setPredicate(cc, pred);
318
319 insert(insn);
320 return insn;
321 }
322
323 void
mkClobber(DataFile f,uint32_t rMask,int unit)324 BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
325 {
326 static const uint16_t baseSize2[16] =
327 {
328 0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
329 0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
330 };
331
332 int base = 0;
333
334 for (; rMask; rMask >>= 4, base += 4) {
335 const uint32_t mask = rMask & 0xf;
336 if (!mask)
337 continue;
338 int base1 = (baseSize2[mask] >> 0) & 0xf;
339 int size1 = (baseSize2[mask] >> 4) & 0xf;
340 int base2 = (baseSize2[mask] >> 8) & 0xf;
341 int size2 = (baseSize2[mask] >> 12) & 0xf;
342 Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
343 if (true) { // size1 can't be 0
344 LValue *reg = new_LValue(func, f);
345 reg->reg.size = size1 << unit;
346 reg->reg.data.id = base + base1;
347 insn->setDef(0, reg);
348 }
349 if (size2) {
350 LValue *reg = new_LValue(func, f);
351 reg->reg.size = size2 << unit;
352 reg->reg.data.id = base + base2;
353 insn->setDef(1, reg);
354 }
355 }
356 }
357
358 ImmediateValue *
mkImm(uint16_t u)359 BuildUtil::mkImm(uint16_t u)
360 {
361 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
362
363 imm->reg.size = 2;
364 imm->reg.type = TYPE_U16;
365 imm->reg.data.u32 = u;
366
367 return imm;
368 }
369
370 ImmediateValue *
mkImm(uint32_t u)371 BuildUtil::mkImm(uint32_t u)
372 {
373 unsigned int pos = u32Hash(u);
374
375 while (imms[pos] && imms[pos]->reg.data.u32 != u)
376 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
377
378 ImmediateValue *imm = imms[pos];
379 if (!imm) {
380 imm = new_ImmediateValue(prog, u);
381 addImmediate(imm);
382 }
383 return imm;
384 }
385
386 ImmediateValue *
mkImm(uint64_t u)387 BuildUtil::mkImm(uint64_t u)
388 {
389 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
390
391 imm->reg.size = 8;
392 imm->reg.type = TYPE_U64;
393 imm->reg.data.u64 = u;
394
395 return imm;
396 }
397
398 ImmediateValue *
mkImm(float f)399 BuildUtil::mkImm(float f)
400 {
401 union {
402 float f32;
403 uint32_t u32;
404 } u;
405 u.f32 = f;
406 return mkImm(u.u32);
407 }
408
409 ImmediateValue *
mkImm(double d)410 BuildUtil::mkImm(double d)
411 {
412 return new_ImmediateValue(prog, d);
413 }
414
415 Value *
loadImm(Value * dst,float f)416 BuildUtil::loadImm(Value *dst, float f)
417 {
418 return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
419 }
420
421 Value *
loadImm(Value * dst,double d)422 BuildUtil::loadImm(Value *dst, double d)
423 {
424 return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d));
425 }
426
427 Value *
loadImm(Value * dst,uint16_t u)428 BuildUtil::loadImm(Value *dst, uint16_t u)
429 {
430 return mkOp1v(OP_MOV, TYPE_U16, dst ? dst : getScratch(2), mkImm(u));
431 }
432
433 Value *
loadImm(Value * dst,uint32_t u)434 BuildUtil::loadImm(Value *dst, uint32_t u)
435 {
436 return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
437 }
438
439 Value *
loadImm(Value * dst,uint64_t u)440 BuildUtil::loadImm(Value *dst, uint64_t u)
441 {
442 return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
443 }
444
445 Symbol *
mkSymbol(DataFile file,int8_t fileIndex,DataType ty,uint32_t baseAddr)446 BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
447 uint32_t baseAddr)
448 {
449 Symbol *sym = new_Symbol(prog, file, fileIndex);
450
451 sym->setOffset(baseAddr);
452 sym->reg.type = ty;
453 sym->reg.size = typeSizeof(ty);
454
455 return sym;
456 }
457
458 Symbol *
mkSysVal(SVSemantic svName,uint32_t svIndex)459 BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
460 {
461 Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
462
463 assert(svIndex < 4 || svName == SV_CLIP_DISTANCE);
464
465 switch (svName) {
466 case SV_POSITION:
467 case SV_FACE:
468 case SV_YDIR:
469 case SV_POINT_SIZE:
470 case SV_POINT_COORD:
471 case SV_CLIP_DISTANCE:
472 case SV_TESS_OUTER:
473 case SV_TESS_INNER:
474 case SV_TESS_COORD:
475 sym->reg.type = TYPE_F32;
476 break;
477 default:
478 sym->reg.type = TYPE_U32;
479 break;
480 }
481 sym->reg.size = typeSizeof(sym->reg.type);
482
483 sym->reg.data.sv.sv = svName;
484 sym->reg.data.sv.index = svIndex;
485
486 return sym;
487 }
488
489 Symbol *
mkTSVal(TSSemantic tsName)490 BuildUtil::mkTSVal(TSSemantic tsName)
491 {
492 Symbol *sym = new_Symbol(prog, FILE_THREAD_STATE, 0);
493 sym->reg.type = TYPE_U32;
494 sym->reg.size = typeSizeof(sym->reg.type);
495 sym->reg.data.ts = tsName;
496 return sym;
497 }
498
499 void
setup(unsigned array,unsigned arrayIdx,uint32_t base,int len,int vecDim,int eltSize,DataFile file,int8_t fileIdx)500 BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx,
501 uint32_t base, int len, int vecDim, int eltSize,
502 DataFile file, int8_t fileIdx)
503 {
504 this->array = array;
505 this->arrayIdx = arrayIdx;
506 this->baseAddr = base;
507 this->arrayLen = len;
508 this->vecDim = vecDim;
509 this->eltSize = eltSize;
510 this->file = file;
511 this->regOnly = !isMemoryFile(file);
512
513 if (!regOnly) {
514 baseSym = new_Symbol(up->getProgram(), file, fileIdx);
515 baseSym->setOffset(baseAddr);
516 baseSym->reg.size = eltSize;
517 } else {
518 baseSym = NULL;
519 }
520 }
521
522 Value *
acquire(ValueMap & m,int i,int c)523 BuildUtil::DataArray::acquire(ValueMap &m, int i, int c)
524 {
525 if (regOnly) {
526 Value *v = lookup(m, i, c);
527 if (!v)
528 v = insert(m, i, c, new_LValue(up->getFunction(), file));
529
530 return v;
531 } else {
532 return up->getScratch(eltSize);
533 }
534 }
535
536 Value *
load(ValueMap & m,int i,int c,Value * ptr)537 BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr)
538 {
539 if (regOnly) {
540 Value *v = lookup(m, i, c);
541 if (!v)
542 v = insert(m, i, c, new_LValue(up->getFunction(), file));
543
544 return v;
545 } else {
546 Value *sym = lookup(m, i, c);
547 if (!sym)
548 sym = insert(m, i, c, mkSymbol(i, c));
549
550 return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr);
551 }
552 }
553
554 void
store(ValueMap & m,int i,int c,Value * ptr,Value * value)555 BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value)
556 {
557 if (regOnly) {
558 assert(!ptr);
559 if (!lookup(m, i, c))
560 insert(m, i, c, value);
561
562 assert(lookup(m, i, c) == value);
563 } else {
564 Value *sym = lookup(m, i, c);
565 if (!sym)
566 sym = insert(m, i, c, mkSymbol(i, c));
567
568 const DataType stTy = typeOfSize(value->reg.size);
569
570 up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value);
571 }
572 }
573
574 Symbol *
mkSymbol(int i,int c)575 BuildUtil::DataArray::mkSymbol(int i, int c)
576 {
577 const unsigned int idx = i * vecDim + c;
578 Symbol *sym = new_Symbol(up->getProgram(), file, 0);
579
580 assert(baseSym || (idx < arrayLen && c < vecDim));
581
582 sym->reg.size = eltSize;
583 sym->reg.type = typeOfSize(eltSize);
584 sym->setAddress(baseSym, baseAddr + idx * eltSize);
585 return sym;
586 }
587
588
589 Instruction *
split64BitOpPostRA(Function * fn,Instruction * i,Value * zero,Value * carry)590 BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
591 Value *zero,
592 Value *carry)
593 {
594 DataType hTy;
595 int srcNr;
596
597 switch (i->dType) {
598 case TYPE_U64: hTy = TYPE_U32; break;
599 case TYPE_S64: hTy = TYPE_S32; break;
600 case TYPE_F64:
601 if (i->op == OP_MOV) {
602 hTy = TYPE_U32;
603 break;
604 }
605 FALLTHROUGH;
606 default:
607 return NULL;
608 }
609
610 switch (i->op) {
611 case OP_MOV: srcNr = 1; break;
612 case OP_ADD:
613 case OP_SUB:
614 if (!carry)
615 return NULL;
616 srcNr = 2;
617 break;
618 case OP_SELP: srcNr = 3; break;
619 default:
620 // TODO when needed
621 return NULL;
622 }
623
624 i->setType(hTy);
625 i->setDef(0, cloneShallow(fn, i->getDef(0)));
626 i->getDef(0)->reg.size = 4;
627 Instruction *lo = i;
628 Instruction *hi = cloneForward(fn, i);
629 lo->bb->insertAfter(lo, hi);
630
631 hi->getDef(0)->reg.data.id++;
632
633 for (int s = 0; s < srcNr; ++s) {
634 if (lo->getSrc(s)->reg.size < 8) {
635 if (s == 2)
636 hi->setSrc(s, lo->getSrc(s));
637 else
638 hi->setSrc(s, zero);
639 } else {
640 if (lo->getSrc(s)->refCount() > 1)
641 lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
642 lo->getSrc(s)->reg.size /= 2;
643 hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
644
645 switch (hi->src(s).getFile()) {
646 case FILE_IMMEDIATE:
647 hi->getSrc(s)->reg.data.u64 >>= 32;
648 break;
649 case FILE_MEMORY_CONST:
650 case FILE_MEMORY_SHARED:
651 case FILE_SHADER_INPUT:
652 case FILE_SHADER_OUTPUT:
653 hi->getSrc(s)->reg.data.offset += 4;
654 break;
655 default:
656 assert(hi->src(s).getFile() == FILE_GPR);
657 hi->getSrc(s)->reg.data.id++;
658 break;
659 }
660 }
661 }
662 if (srcNr == 2) {
663 lo->setFlagsDef(1, carry);
664 hi->setFlagsSrc(hi->srcCount(), carry);
665 }
666 return hi;
667 }
668
669 } // namespace nv50_ir
670