1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 namespace nv50_ir {
27
BuildUtil()28 BuildUtil::BuildUtil()
29 {
30 init(NULL);
31 }
32
BuildUtil(Program * prog)33 BuildUtil::BuildUtil(Program *prog)
34 {
35 init(prog);
36 }
37
38 void
init(Program * prog)39 BuildUtil::init(Program *prog)
40 {
41 this->prog = prog;
42
43 func = NULL;
44 bb = NULL;
45 pos = NULL;
46
47 memset(imms, 0, sizeof(imms));
48 immCount = 0;
49 }
50
51 void
addImmediate(ImmediateValue * imm)52 BuildUtil::addImmediate(ImmediateValue *imm)
53 {
54 if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
55 return;
56
57 unsigned int pos = u32Hash(imm->reg.data.u32);
58
59 while (imms[pos])
60 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
61 imms[pos] = imm;
62 immCount++;
63 }
64
65 Instruction *
mkOp1(operation op,DataType ty,Value * dst,Value * src)66 BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
67 {
68 Instruction *insn = new_Instruction(func, op, ty);
69
70 insn->setDef(0, dst);
71 insn->setSrc(0, src);
72
73 insert(insn);
74 return insn;
75 }
76
77 Instruction *
mkOp2(operation op,DataType ty,Value * dst,Value * src0,Value * src1)78 BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
79 Value *src0, Value *src1)
80 {
81 Instruction *insn = new_Instruction(func, op, ty);
82
83 insn->setDef(0, dst);
84 insn->setSrc(0, src0);
85 insn->setSrc(1, src1);
86
87 insert(insn);
88 return insn;
89 }
90
91 Instruction *
mkOp3(operation op,DataType ty,Value * dst,Value * src0,Value * src1,Value * src2)92 BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
93 Value *src0, Value *src1, Value *src2)
94 {
95 Instruction *insn = new_Instruction(func, op, ty);
96
97 insn->setDef(0, dst);
98 insn->setSrc(0, src0);
99 insn->setSrc(1, src1);
100 insn->setSrc(2, src2);
101
102 insert(insn);
103 return insn;
104 }
105
106 Instruction *
mkLoad(DataType ty,Value * dst,Symbol * mem,Value * ptr)107 BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
108 {
109 Instruction *insn = new_Instruction(func, OP_LOAD, ty);
110
111 insn->setDef(0, dst);
112 insn->setSrc(0, mem);
113 if (ptr)
114 insn->setIndirect(0, 0, ptr);
115
116 insert(insn);
117 return insn;
118 }
119
120 Instruction *
mkStore(operation op,DataType ty,Symbol * mem,Value * ptr,Value * stVal)121 BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
122 Value *stVal)
123 {
124 Instruction *insn = new_Instruction(func, op, ty);
125
126 insn->setSrc(0, mem);
127 insn->setSrc(1, stVal);
128 if (ptr)
129 insn->setIndirect(0, 0, ptr);
130
131 insert(insn);
132 return insn;
133 }
134
135 Instruction *
mkFetch(Value * dst,DataType ty,DataFile file,int32_t offset,Value * attrRel,Value * primRel)136 BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
137 Value *attrRel, Value *primRel)
138 {
139 Symbol *sym = mkSymbol(file, 0, ty, offset);
140
141 Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
142
143 insn->setIndirect(0, 0, attrRel);
144 insn->setIndirect(0, 1, primRel);
145
146 // already inserted
147 return insn;
148 }
149
150 Instruction *
mkInterp(unsigned mode,Value * dst,int32_t offset,Value * rel)151 BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
152 {
153 operation op = OP_LINTERP;
154 DataType ty = TYPE_F32;
155
156 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
157 ty = TYPE_U32;
158 else
159 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
160 op = OP_PINTERP;
161
162 Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
163
164 Instruction *insn = mkOp1(op, ty, dst, sym);
165 insn->setIndirect(0, 0, rel);
166 insn->setInterpolate(mode);
167 return insn;
168 }
169
170 Instruction *
mkMov(Value * dst,Value * src,DataType ty)171 BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
172 {
173 Instruction *insn = new_Instruction(func, OP_MOV, ty);
174
175 insn->setDef(0, dst);
176 insn->setSrc(0, src);
177
178 insert(insn);
179 return insn;
180 }
181
182 Instruction *
mkMovToReg(int id,Value * src)183 BuildUtil::mkMovToReg(int id, Value *src)
184 {
185 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
186
187 insn->setDef(0, new_LValue(func, FILE_GPR));
188 insn->getDef(0)->reg.data.id = id;
189 insn->setSrc(0, src);
190
191 insert(insn);
192 return insn;
193 }
194
195 Instruction *
mkMovFromReg(Value * dst,int id)196 BuildUtil::mkMovFromReg(Value *dst, int id)
197 {
198 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
199
200 insn->setDef(0, dst);
201 insn->setSrc(0, new_LValue(func, FILE_GPR));
202 insn->getSrc(0)->reg.data.id = id;
203
204 insert(insn);
205 return insn;
206 }
207
208 Instruction *
mkCvt(operation op,DataType dstTy,Value * dst,DataType srcTy,Value * src)209 BuildUtil::mkCvt(operation op,
210 DataType dstTy, Value *dst, DataType srcTy, Value *src)
211 {
212 Instruction *insn = new_Instruction(func, op, dstTy);
213
214 insn->setType(dstTy, srcTy);
215 insn->setDef(0, dst);
216 insn->setSrc(0, src);
217
218 insert(insn);
219 return insn;
220 }
221
222 CmpInstruction *
mkCmp(operation op,CondCode cc,DataType dstTy,Value * dst,DataType srcTy,Value * src0,Value * src1,Value * src2)223 BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst,
224 DataType srcTy, Value *src0, Value *src1, Value *src2)
225 {
226 CmpInstruction *insn = new_CmpInstruction(func, op);
227
228 insn->setType((dst->reg.file == FILE_PREDICATE ||
229 dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy);
230 insn->setCondition(cc);
231 insn->setDef(0, dst);
232 insn->setSrc(0, src0);
233 insn->setSrc(1, src1);
234 if (src2)
235 insn->setSrc(2, src2);
236
237 if (dst->reg.file == FILE_FLAGS)
238 insn->flagsDef = 0;
239
240 insert(insn);
241 return insn;
242 }
243
244 TexInstruction *
mkTex(operation op,TexTarget targ,uint16_t tic,uint16_t tsc,const std::vector<Value * > & def,const std::vector<Value * > & src)245 BuildUtil::mkTex(operation op, TexTarget targ,
246 uint16_t tic, uint16_t tsc,
247 const std::vector<Value *> &def,
248 const std::vector<Value *> &src)
249 {
250 TexInstruction *tex = new_TexInstruction(func, op);
251
252 for (size_t d = 0; d < def.size() && def[d]; ++d)
253 tex->setDef(d, def[d]);
254 for (size_t s = 0; s < src.size() && src[s]; ++s)
255 tex->setSrc(s, src[s]);
256
257 tex->setTexture(targ, tic, tsc);
258
259 insert(tex);
260 return tex;
261 }
262
263 Instruction *
mkQuadop(uint8_t q,Value * def,uint8_t l,Value * src0,Value * src1)264 BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
265 {
266 Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
267 quadop->subOp = q;
268 quadop->lanes = l;
269 return quadop;
270 }
271
272 Instruction *
mkSelect(Value * pred,Value * dst,Value * trSrc,Value * flSrc)273 BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
274 {
275 LValue *def0 = getSSA();
276 LValue *def1 = getSSA();
277
278 mkMov(def0, trSrc)->setPredicate(CC_P, pred);
279 mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
280
281 return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
282 }
283
284 Instruction *
mkSplit(Value * h[2],uint8_t halfSize,Value * val)285 BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
286 {
287 Instruction *insn = NULL;
288
289 const DataType fTy = typeOfSize(halfSize * 2);
290
291 if (val->reg.file == FILE_IMMEDIATE)
292 val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
293
294 if (isMemoryFile(val->reg.file)) {
295 h[0] = cloneShallow(getFunction(), val);
296 h[1] = cloneShallow(getFunction(), val);
297 h[0]->reg.size = halfSize;
298 h[1]->reg.size = halfSize;
299 h[1]->reg.data.offset += halfSize;
300 } else {
301 h[0] = getSSA(halfSize, val->reg.file);
302 h[1] = getSSA(halfSize, val->reg.file);
303 insn = mkOp1(OP_SPLIT, fTy, h[0], val);
304 insn->setDef(1, h[1]);
305 }
306 return insn;
307 }
308
309 FlowInstruction *
mkFlow(operation op,void * targ,CondCode cc,Value * pred)310 BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
311 {
312 FlowInstruction *insn = new_FlowInstruction(func, op, targ);
313
314 if (pred)
315 insn->setPredicate(cc, pred);
316
317 insert(insn);
318 return insn;
319 }
320
321 void
mkClobber(DataFile f,uint32_t rMask,int unit)322 BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
323 {
324 static const uint16_t baseSize2[16] =
325 {
326 0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
327 0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
328 };
329
330 int base = 0;
331
332 for (; rMask; rMask >>= 4, base += 4) {
333 const uint32_t mask = rMask & 0xf;
334 if (!mask)
335 continue;
336 int base1 = (baseSize2[mask] >> 0) & 0xf;
337 int size1 = (baseSize2[mask] >> 4) & 0xf;
338 int base2 = (baseSize2[mask] >> 8) & 0xf;
339 int size2 = (baseSize2[mask] >> 12) & 0xf;
340 Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
341 if (1) { // size1 can't be 0
342 LValue *reg = new_LValue(func, f);
343 reg->reg.size = size1 << unit;
344 reg->reg.data.id = base + base1;
345 insn->setDef(0, reg);
346 }
347 if (size2) {
348 LValue *reg = new_LValue(func, f);
349 reg->reg.size = size2 << unit;
350 reg->reg.data.id = base + base2;
351 insn->setDef(1, reg);
352 }
353 }
354 }
355
356 ImmediateValue *
mkImm(uint32_t u)357 BuildUtil::mkImm(uint32_t u)
358 {
359 unsigned int pos = u32Hash(u);
360
361 while (imms[pos] && imms[pos]->reg.data.u32 != u)
362 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
363
364 ImmediateValue *imm = imms[pos];
365 if (!imm) {
366 imm = new_ImmediateValue(prog, u);
367 addImmediate(imm);
368 }
369 return imm;
370 }
371
372 ImmediateValue *
mkImm(uint64_t u)373 BuildUtil::mkImm(uint64_t u)
374 {
375 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
376
377 imm->reg.size = 8;
378 imm->reg.type = TYPE_U64;
379 imm->reg.data.u64 = u;
380
381 return imm;
382 }
383
384 ImmediateValue *
mkImm(float f)385 BuildUtil::mkImm(float f)
386 {
387 union {
388 float f32;
389 uint32_t u32;
390 } u;
391 u.f32 = f;
392 return mkImm(u.u32);
393 }
394
395 ImmediateValue *
mkImm(double d)396 BuildUtil::mkImm(double d)
397 {
398 return new_ImmediateValue(prog, d);
399 }
400
401 Value *
loadImm(Value * dst,float f)402 BuildUtil::loadImm(Value *dst, float f)
403 {
404 return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
405 }
406
407 Value *
loadImm(Value * dst,double d)408 BuildUtil::loadImm(Value *dst, double d)
409 {
410 return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d));
411 }
412
413 Value *
loadImm(Value * dst,uint32_t u)414 BuildUtil::loadImm(Value *dst, uint32_t u)
415 {
416 return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
417 }
418
419 Value *
loadImm(Value * dst,uint64_t u)420 BuildUtil::loadImm(Value *dst, uint64_t u)
421 {
422 return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
423 }
424
425 Symbol *
mkSymbol(DataFile file,int8_t fileIndex,DataType ty,uint32_t baseAddr)426 BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
427 uint32_t baseAddr)
428 {
429 Symbol *sym = new_Symbol(prog, file, fileIndex);
430
431 sym->setOffset(baseAddr);
432 sym->reg.type = ty;
433 sym->reg.size = typeSizeof(ty);
434
435 return sym;
436 }
437
438 Symbol *
mkSysVal(SVSemantic svName,uint32_t svIndex)439 BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
440 {
441 Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
442
443 assert(svIndex < 4 || svName == SV_CLIP_DISTANCE);
444
445 switch (svName) {
446 case SV_POSITION:
447 case SV_FACE:
448 case SV_YDIR:
449 case SV_POINT_SIZE:
450 case SV_POINT_COORD:
451 case SV_CLIP_DISTANCE:
452 case SV_TESS_OUTER:
453 case SV_TESS_INNER:
454 case SV_TESS_COORD:
455 sym->reg.type = TYPE_F32;
456 break;
457 default:
458 sym->reg.type = TYPE_U32;
459 break;
460 }
461 sym->reg.size = typeSizeof(sym->reg.type);
462
463 sym->reg.data.sv.sv = svName;
464 sym->reg.data.sv.index = svIndex;
465
466 return sym;
467 }
468
469 void
setup(unsigned array,unsigned arrayIdx,uint32_t base,int len,int vecDim,int eltSize,DataFile file,int8_t fileIdx)470 BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx,
471 uint32_t base, int len, int vecDim, int eltSize,
472 DataFile file, int8_t fileIdx)
473 {
474 this->array = array;
475 this->arrayIdx = arrayIdx;
476 this->baseAddr = base;
477 this->arrayLen = len;
478 this->vecDim = vecDim;
479 this->eltSize = eltSize;
480 this->file = file;
481 this->regOnly = !isMemoryFile(file);
482
483 if (!regOnly) {
484 baseSym = new_Symbol(up->getProgram(), file, fileIdx);
485 baseSym->setOffset(baseAddr);
486 baseSym->reg.size = eltSize;
487 } else {
488 baseSym = NULL;
489 }
490 }
491
492 Value *
acquire(ValueMap & m,int i,int c)493 BuildUtil::DataArray::acquire(ValueMap &m, int i, int c)
494 {
495 if (regOnly) {
496 Value *v = lookup(m, i, c);
497 if (!v)
498 v = insert(m, i, c, new_LValue(up->getFunction(), file));
499
500 return v;
501 } else {
502 return up->getScratch(eltSize);
503 }
504 }
505
506 Value *
load(ValueMap & m,int i,int c,Value * ptr)507 BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr)
508 {
509 if (regOnly) {
510 Value *v = lookup(m, i, c);
511 if (!v)
512 v = insert(m, i, c, new_LValue(up->getFunction(), file));
513
514 return v;
515 } else {
516 Value *sym = lookup(m, i, c);
517 if (!sym)
518 sym = insert(m, i, c, mkSymbol(i, c));
519
520 return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr);
521 }
522 }
523
524 void
store(ValueMap & m,int i,int c,Value * ptr,Value * value)525 BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value)
526 {
527 if (regOnly) {
528 assert(!ptr);
529 if (!lookup(m, i, c))
530 insert(m, i, c, value);
531
532 assert(lookup(m, i, c) == value);
533 } else {
534 Value *sym = lookup(m, i, c);
535 if (!sym)
536 sym = insert(m, i, c, mkSymbol(i, c));
537
538 const DataType stTy = typeOfSize(value->reg.size);
539
540 up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value);
541 }
542 }
543
544 Symbol *
mkSymbol(int i,int c)545 BuildUtil::DataArray::mkSymbol(int i, int c)
546 {
547 const unsigned int idx = i * vecDim + c;
548 Symbol *sym = new_Symbol(up->getProgram(), file, 0);
549
550 assert(baseSym || (idx < arrayLen && c < vecDim));
551
552 sym->reg.size = eltSize;
553 sym->reg.type = typeOfSize(eltSize);
554 sym->setAddress(baseSym, baseAddr + idx * eltSize);
555 return sym;
556 }
557
558
559 Instruction *
split64BitOpPostRA(Function * fn,Instruction * i,Value * zero,Value * carry)560 BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
561 Value *zero,
562 Value *carry)
563 {
564 DataType hTy;
565 int srcNr;
566
567 switch (i->dType) {
568 case TYPE_U64: hTy = TYPE_U32; break;
569 case TYPE_S64: hTy = TYPE_S32; break;
570 case TYPE_F64:
571 if (i->op == OP_MOV) {
572 hTy = TYPE_U32;
573 break;
574 }
575 /* fallthrough */
576 default:
577 return NULL;
578 }
579
580 switch (i->op) {
581 case OP_MOV: srcNr = 1; break;
582 case OP_ADD:
583 case OP_SUB:
584 if (!carry)
585 return NULL;
586 srcNr = 2;
587 break;
588 case OP_SELP: srcNr = 3; break;
589 default:
590 // TODO when needed
591 return NULL;
592 }
593
594 i->setType(hTy);
595 i->setDef(0, cloneShallow(fn, i->getDef(0)));
596 i->getDef(0)->reg.size = 4;
597 Instruction *lo = i;
598 Instruction *hi = cloneForward(fn, i);
599 lo->bb->insertAfter(lo, hi);
600
601 hi->getDef(0)->reg.data.id++;
602
603 for (int s = 0; s < srcNr; ++s) {
604 if (lo->getSrc(s)->reg.size < 8) {
605 if (s == 2)
606 hi->setSrc(s, lo->getSrc(s));
607 else
608 hi->setSrc(s, zero);
609 } else {
610 if (lo->getSrc(s)->refCount() > 1)
611 lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
612 lo->getSrc(s)->reg.size /= 2;
613 hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
614
615 switch (hi->src(s).getFile()) {
616 case FILE_IMMEDIATE:
617 hi->getSrc(s)->reg.data.u64 >>= 32;
618 break;
619 case FILE_MEMORY_CONST:
620 case FILE_MEMORY_SHARED:
621 case FILE_SHADER_INPUT:
622 case FILE_SHADER_OUTPUT:
623 hi->getSrc(s)->reg.data.offset += 4;
624 break;
625 default:
626 assert(hi->src(s).getFile() == FILE_GPR);
627 hi->getSrc(s)->reg.data.id++;
628 break;
629 }
630 }
631 }
632 if (srcNr == 2) {
633 lo->setFlagsDef(1, carry);
634 hi->setFlagsSrc(hi->srcCount(), carry);
635 }
636 return hi;
637 }
638
639 } // namespace nv50_ir
640