1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 #include "codegen/nv50_ir_target_nvc0.h"
27 #include "codegen/nv50_ir_lowering_nvc0.h"
28
29 #include <limits>
30
31 namespace nv50_ir {
32
33 #define QOP_ADD 0
34 #define QOP_SUBR 1
35 #define QOP_SUB 2
36 #define QOP_MOV2 3
37
38 // UL UR LL LR
39 #define QUADOP(q, r, s, t) \
40 ((QOP_##q << 6) | (QOP_##r << 4) | \
41 (QOP_##s << 2) | (QOP_##t << 0))
42
43 void
handleDIV(Instruction * i)44 NVC0LegalizeSSA::handleDIV(Instruction *i)
45 {
46 FlowInstruction *call;
47 int builtin;
48
49 bld.setPosition(i, false);
50
51 // Generate movs to the input regs for the call we want to generate
52 for (int s = 0; i->srcExists(s); ++s) {
53 Instruction *ld = i->getSrc(s)->getInsn();
54 // check if we are moving an immediate, propagate it in that case
55 if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV) ||
56 !(ld->src(0).getFile() == FILE_IMMEDIATE))
57 bld.mkMovToReg(s, i->getSrc(s));
58 else {
59 assert(ld->getSrc(0) != NULL);
60 bld.mkMovToReg(s, ld->getSrc(0));
61 // Clear the src, to make code elimination possible here before we
62 // delete the instruction i later
63 i->setSrc(s, NULL);
64 if (ld->isDead())
65 delete_Instruction(prog, ld);
66 }
67 }
68
69 switch (i->dType) {
70 case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;
71 case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;
72 default:
73 return;
74 }
75 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
76 bld.mkMovFromReg(i->getDef(0), i->op == OP_DIV ? 0 : 1);
77 bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);
78 bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);
79
80 call->fixed = 1;
81 call->absolute = call->builtin = 1;
82 call->target.builtin = builtin;
83 delete_Instruction(prog, i);
84 }
85
86 void
handleRCPRSQLib(Instruction * i,Value * src[])87 NVC0LegalizeSSA::handleRCPRSQLib(Instruction *i, Value *src[])
88 {
89 FlowInstruction *call;
90 Value *def[2];
91 int builtin;
92
93 def[0] = bld.mkMovToReg(0, src[0])->getDef(0);
94 def[1] = bld.mkMovToReg(1, src[1])->getDef(0);
95
96 if (i->op == OP_RCP)
97 builtin = NVC0_BUILTIN_RCP_F64;
98 else
99 builtin = NVC0_BUILTIN_RSQ_F64;
100
101 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
102 def[0] = bld.getSSA();
103 def[1] = bld.getSSA();
104 bld.mkMovFromReg(def[0], 0);
105 bld.mkMovFromReg(def[1], 1);
106 bld.mkClobber(FILE_GPR, 0x3fc, 2);
107 bld.mkClobber(FILE_PREDICATE, i->op == OP_RSQ ? 0x3 : 0x1, 0);
108 bld.mkOp2(OP_MERGE, TYPE_U64, i->getDef(0), def[0], def[1]);
109
110 call->fixed = 1;
111 call->absolute = call->builtin = 1;
112 call->target.builtin = builtin;
113 delete_Instruction(prog, i);
114
115 prog->fp64 = true;
116 }
117
118 void
handleRCPRSQ(Instruction * i)119 NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)
120 {
121 assert(i->dType == TYPE_F64);
122 // There are instructions that will compute the high 32 bits of the 64-bit
123 // float. We will just stick 0 in the bottom 32 bits.
124
125 bld.setPosition(i, false);
126
127 // 1. Take the source and it up.
128 Value *src[2], *dst[2], *def = i->getDef(0);
129 bld.mkSplit(src, 4, i->getSrc(0));
130
131 int chip = prog->getTarget()->getChipset();
132 if (chip >= NVISA_GK104_CHIPSET) {
133 handleRCPRSQLib(i, src);
134 return;
135 }
136
137 // 2. We don't care about the low 32 bits of the destination. Stick a 0 in.
138 dst[0] = bld.loadImm(NULL, 0);
139 dst[1] = bld.getSSA();
140
141 // 3. The new version of the instruction takes the high 32 bits of the
142 // source and outputs the high 32 bits of the destination.
143 i->setSrc(0, src[1]);
144 i->setDef(0, dst[1]);
145 i->setType(TYPE_F32);
146 i->subOp = NV50_IR_SUBOP_RCPRSQ_64H;
147
148 // 4. Recombine the two dst pieces back into the original destination.
149 bld.setPosition(i, true);
150 bld.mkOp2(OP_MERGE, TYPE_U64, def, dst[0], dst[1]);
151 }
152
153 void
handleFTZ(Instruction * i)154 NVC0LegalizeSSA::handleFTZ(Instruction *i)
155 {
156 // Only want to flush float inputs
157 assert(i->sType == TYPE_F32);
158
159 // If we're already flushing denorms (and NaN's) to zero, no need for this.
160 if (i->dnz)
161 return;
162
163 // Only certain classes of operations can flush
164 OpClass cls = prog->getTarget()->getOpClass(i->op);
165 if (cls != OPCLASS_ARITH && cls != OPCLASS_COMPARE &&
166 cls != OPCLASS_CONVERT)
167 return;
168
169 i->ftz = true;
170 }
171
172 void
handleTEXLOD(TexInstruction * i)173 NVC0LegalizeSSA::handleTEXLOD(TexInstruction *i)
174 {
175 if (i->tex.levelZero)
176 return;
177
178 ImmediateValue lod;
179
180 // The LOD argument comes right after the coordinates (before depth bias,
181 // offsets, etc).
182 int arg = i->tex.target.getArgCount();
183
184 // SM30+ stores the indirect handle as a separate arg, which comes before
185 // the LOD.
186 if (prog->getTarget()->getChipset() >= NVISA_GK104_CHIPSET &&
187 i->tex.rIndirectSrc >= 0)
188 arg++;
189 // SM20 stores indirect handle combined with array coordinate
190 if (prog->getTarget()->getChipset() < NVISA_GK104_CHIPSET &&
191 !i->tex.target.isArray() &&
192 i->tex.rIndirectSrc >= 0)
193 arg++;
194
195 if (!i->src(arg).getImmediate(lod) || !lod.isInteger(0))
196 return;
197
198 if (i->op == OP_TXL)
199 i->op = OP_TEX;
200 i->tex.levelZero = true;
201 i->moveSources(arg + 1, -1);
202 }
203
204 void
handleShift(Instruction * lo)205 NVC0LegalizeSSA::handleShift(Instruction *lo)
206 {
207 Value *shift = lo->getSrc(1);
208 Value *dst64 = lo->getDef(0);
209 Value *src[2], *dst[2];
210 operation op = lo->op;
211
212 bld.setPosition(lo, false);
213
214 bld.mkSplit(src, 4, lo->getSrc(0));
215
216 // SM30 and prior don't have the fancy new SHF.L/R ops. So the logic has to
217 // be completely emulated. For SM35+, we can use the more directed SHF
218 // operations.
219 if (prog->getTarget()->getChipset() < NVISA_GK20A_CHIPSET) {
220 // The strategy here is to handle shifts >= 32 and less than 32 as
221 // separate parts.
222 //
223 // For SHL:
224 // If the shift is <= 32, then
225 // (HI,LO) << x = (HI << x | (LO >> (32 - x)), LO << x)
226 // If the shift is > 32, then
227 // (HI,LO) << x = (LO << (x - 32), 0)
228 //
229 // For SHR:
230 // If the shift is <= 32, then
231 // (HI,LO) >> x = (HI >> x, (HI << (32 - x)) | LO >> x)
232 // If the shift is > 32, then
233 // (HI,LO) >> x = (0, HI >> (x - 32))
234 //
235 // Note that on NVIDIA hardware, a shift > 32 yields a 0 value, which we
236 // can use to our advantage. Also note the structural similarities
237 // between the right/left cases. The main difference is swapping hi/lo
238 // on input and output.
239
240 Value *x32_minus_shift, *pred, *hi1, *hi2;
241 DataType type = isSignedIntType(lo->dType) ? TYPE_S32 : TYPE_U32;
242 operation antiop = op == OP_SHR ? OP_SHL : OP_SHR;
243 if (op == OP_SHR)
244 std::swap(src[0], src[1]);
245 bld.mkOp2(OP_ADD, TYPE_U32, (x32_minus_shift = bld.getSSA()), shift, bld.mkImm(0x20))
246 ->src(0).mod = Modifier(NV50_IR_MOD_NEG);
247 bld.mkCmp(OP_SET, CC_LE, TYPE_U8, (pred = bld.getSSA(1, FILE_PREDICATE)),
248 TYPE_U32, shift, bld.mkImm(32));
249 // Compute HI (shift <= 32)
250 bld.mkOp2(OP_OR, TYPE_U32, (hi1 = bld.getSSA()),
251 bld.mkOp2v(op, TYPE_U32, bld.getSSA(), src[1], shift),
252 bld.mkOp2v(antiop, TYPE_U32, bld.getSSA(), src[0], x32_minus_shift))
253 ->setPredicate(CC_P, pred);
254 // Compute LO (all shift values)
255 bld.mkOp2(op, type, (dst[0] = bld.getSSA()), src[0], shift);
256 // Compute HI (shift > 32)
257 bld.mkOp2(op, type, (hi2 = bld.getSSA()), src[0],
258 bld.mkOp1v(OP_NEG, TYPE_S32, bld.getSSA(), x32_minus_shift))
259 ->setPredicate(CC_NOT_P, pred);
260 bld.mkOp2(OP_UNION, TYPE_U32, (dst[1] = bld.getSSA()), hi1, hi2);
261 if (op == OP_SHR)
262 std::swap(dst[0], dst[1]);
263 bld.mkOp2(OP_MERGE, TYPE_U64, dst64, dst[0], dst[1]);
264 delete_Instruction(prog, lo);
265 return;
266 }
267
268 Instruction *hi = new_Instruction(func, op, TYPE_U32);
269 lo->bb->insertAfter(lo, hi);
270
271 hi->sType = lo->sType;
272 lo->dType = TYPE_U32;
273
274 hi->setDef(0, (dst[1] = bld.getSSA()));
275 if (lo->op == OP_SHR)
276 hi->subOp |= NV50_IR_SUBOP_SHIFT_HIGH;
277 lo->setDef(0, (dst[0] = bld.getSSA()));
278
279 bld.setPosition(hi, true);
280
281 if (lo->op == OP_SHL)
282 std::swap(hi, lo);
283
284 hi->setSrc(0, new_ImmediateValue(prog, 0u));
285 hi->setSrc(1, shift);
286 hi->setSrc(2, lo->op == OP_SHL ? src[0] : src[1]);
287
288 lo->setSrc(0, src[0]);
289 lo->setSrc(1, shift);
290 lo->setSrc(2, src[1]);
291
292 bld.mkOp2(OP_MERGE, TYPE_U64, dst64, dst[0], dst[1]);
293 }
294
295 void
handleSET(CmpInstruction * cmp)296 NVC0LegalizeSSA::handleSET(CmpInstruction *cmp)
297 {
298 DataType hTy = cmp->sType == TYPE_S64 ? TYPE_S32 : TYPE_U32;
299 Value *carry;
300 Value *src0[2], *src1[2];
301 bld.setPosition(cmp, false);
302
303 bld.mkSplit(src0, 4, cmp->getSrc(0));
304 bld.mkSplit(src1, 4, cmp->getSrc(1));
305 bld.mkOp2(OP_SUB, hTy, NULL, src0[0], src1[0])
306 ->setFlagsDef(0, (carry = bld.getSSA(1, FILE_FLAGS)));
307 cmp->setFlagsSrc(cmp->srcCount(), carry);
308 cmp->setSrc(0, src0[1]);
309 cmp->setSrc(1, src1[1]);
310 cmp->sType = hTy;
311 }
312
313 void
handleBREV(Instruction * i)314 NVC0LegalizeSSA::handleBREV(Instruction *i)
315 {
316 i->op = OP_EXTBF;
317 i->subOp = NV50_IR_SUBOP_EXTBF_REV;
318 i->setSrc(1, bld.mkImm(0x2000));
319 }
320
321 bool
visit(Function * fn)322 NVC0LegalizeSSA::visit(Function *fn)
323 {
324 bld.setProgram(fn->getProgram());
325 return true;
326 }
327
328 bool
visit(BasicBlock * bb)329 NVC0LegalizeSSA::visit(BasicBlock *bb)
330 {
331 Instruction *next;
332 for (Instruction *i = bb->getEntry(); i; i = next) {
333 next = i->next;
334
335 if (i->sType == TYPE_F32 && prog->getType() != Program::TYPE_COMPUTE)
336 handleFTZ(i);
337
338 switch (i->op) {
339 case OP_DIV:
340 case OP_MOD:
341 if (i->sType != TYPE_F32)
342 handleDIV(i);
343 break;
344 case OP_RCP:
345 case OP_RSQ:
346 if (i->dType == TYPE_F64)
347 handleRCPRSQ(i);
348 break;
349 case OP_TXL:
350 case OP_TXF:
351 handleTEXLOD(i->asTex());
352 break;
353 case OP_SHR:
354 case OP_SHL:
355 if (typeSizeof(i->sType) == 8)
356 handleShift(i);
357 break;
358 case OP_SET:
359 case OP_SET_AND:
360 case OP_SET_OR:
361 case OP_SET_XOR:
362 if (typeSizeof(i->sType) == 8 && i->sType != TYPE_F64)
363 handleSET(i->asCmp());
364 break;
365 case OP_BREV:
366 handleBREV(i);
367 break;
368 default:
369 break;
370 }
371 }
372 return true;
373 }
374
NVC0LegalizePostRA(const Program * prog)375 NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)
376 : rZero(NULL),
377 carry(NULL),
378 pOne(NULL),
379 needTexBar(prog->getTarget()->getChipset() >= 0xe0 &&
380 prog->getTarget()->getChipset() < 0x110)
381 {
382 }
383
384 bool
insnDominatedBy(const Instruction * later,const Instruction * early) const385 NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,
386 const Instruction *early) const
387 {
388 if (early->bb == later->bb)
389 return early->serial < later->serial;
390 return later->bb->dominatedBy(early->bb);
391 }
392
393 void
addTexUse(std::list<TexUse> & uses,Instruction * usei,const Instruction * texi)394 NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses,
395 Instruction *usei, const Instruction *texi)
396 {
397 bool add = true;
398 bool dominated = insnDominatedBy(usei, texi);
399 // Uses before the tex have to all be included. Just because an earlier
400 // instruction dominates another instruction doesn't mean that there's no
401 // way to get from the tex to the later instruction. For example you could
402 // have nested loops, with the tex in the inner loop, and uses before it in
403 // both loops - even though the outer loop's instruction would dominate the
404 // inner's, we still want a texbar before the inner loop's instruction.
405 //
406 // However we can still use the eliding logic between uses dominated by the
407 // tex instruction, as that is unambiguously correct.
408 if (dominated) {
409 for (std::list<TexUse>::iterator it = uses.begin(); it != uses.end();) {
410 if (it->after) {
411 if (insnDominatedBy(usei, it->insn)) {
412 add = false;
413 break;
414 }
415 if (insnDominatedBy(it->insn, usei)) {
416 it = uses.erase(it);
417 continue;
418 }
419 }
420 ++it;
421 }
422 }
423 if (add)
424 uses.push_back(TexUse(usei, texi, dominated));
425 }
426
427 // While it might be tempting to use the an algorithm that just looks at tex
428 // uses, not all texture results are guaranteed to be used on all paths. In
429 // the case where along some control flow path a texture result is never used,
430 // we might reuse that register for something else, creating a
431 // write-after-write hazard. So we have to manually look through all
432 // instructions looking for ones that reference the registers in question.
433 void
findFirstUses(Instruction * texi,std::list<TexUse> & uses)434 NVC0LegalizePostRA::findFirstUses(
435 Instruction *texi, std::list<TexUse> &uses)
436 {
437 int minGPR = texi->def(0).rep()->reg.data.id;
438 int maxGPR = minGPR + texi->def(0).rep()->reg.size / 4 - 1;
439
440 unordered_set<const BasicBlock *> visited;
441 findFirstUsesBB(minGPR, maxGPR, texi->next, texi, uses, visited);
442 }
443
444 void
findFirstUsesBB(int minGPR,int maxGPR,Instruction * start,const Instruction * texi,std::list<TexUse> & uses,unordered_set<const BasicBlock * > & visited)445 NVC0LegalizePostRA::findFirstUsesBB(
446 int minGPR, int maxGPR, Instruction *start,
447 const Instruction *texi, std::list<TexUse> &uses,
448 unordered_set<const BasicBlock *> &visited)
449 {
450 const BasicBlock *bb = start->bb;
451
452 // We don't process the whole bb the first time around. This is correct,
453 // however we might be in a loop and hit this BB again, and need to process
454 // the full thing. So only mark a bb as visited if we processed it from the
455 // beginning.
456 if (start == bb->getEntry()) {
457 if (visited.find(bb) != visited.end())
458 return;
459 visited.insert(bb);
460 }
461
462 for (Instruction *insn = start; insn != bb->getExit(); insn = insn->next) {
463 if (insn->isNop())
464 continue;
465
466 for (int d = 0; insn->defExists(d); ++d) {
467 const Value *def = insn->def(d).rep();
468 if (insn->def(d).getFile() != FILE_GPR ||
469 def->reg.data.id + def->reg.size / 4 - 1 < minGPR ||
470 def->reg.data.id > maxGPR)
471 continue;
472 addTexUse(uses, insn, texi);
473 return;
474 }
475
476 for (int s = 0; insn->srcExists(s); ++s) {
477 const Value *src = insn->src(s).rep();
478 if (insn->src(s).getFile() != FILE_GPR ||
479 src->reg.data.id + src->reg.size / 4 - 1 < minGPR ||
480 src->reg.data.id > maxGPR)
481 continue;
482 addTexUse(uses, insn, texi);
483 return;
484 }
485 }
486
487 for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
488 findFirstUsesBB(minGPR, maxGPR, BasicBlock::get(ei.getNode())->getEntry(),
489 texi, uses, visited);
490 }
491 }
492
493 // Texture barriers:
494 // This pass is a bit long and ugly and can probably be optimized.
495 //
496 // 1. obtain a list of TEXes and their outputs' first use(s)
497 // 2. calculate the barrier level of each first use (minimal number of TEXes,
498 // over all paths, between the TEX and the use in question)
499 // 3. for each barrier, if all paths from the source TEX to that barrier
500 // contain a barrier of lesser level, it can be culled
501 bool
insertTextureBarriers(Function * fn)502 NVC0LegalizePostRA::insertTextureBarriers(Function *fn)
503 {
504 std::list<TexUse> *uses;
505 std::vector<Instruction *> texes;
506 std::vector<int> bbFirstTex;
507 std::vector<int> bbFirstUse;
508 std::vector<int> texCounts;
509 std::vector<TexUse> useVec;
510 ArrayList insns;
511
512 fn->orderInstructions(insns);
513
514 texCounts.resize(fn->allBBlocks.getSize(), 0);
515 bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());
516 bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());
517
518 // tag BB CFG nodes by their id for later
519 for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {
520 BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get());
521 if (bb)
522 bb->cfg.tag = bb->getId();
523 }
524
525 // gather the first uses for each TEX
526 for (int i = 0; i < insns.getSize(); ++i) {
527 Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i));
528 if (isTextureOp(tex->op)) {
529 texes.push_back(tex);
530 if (!texCounts.at(tex->bb->getId()))
531 bbFirstTex[tex->bb->getId()] = texes.size() - 1;
532 texCounts[tex->bb->getId()]++;
533 }
534 }
535 insns.clear();
536 if (texes.empty())
537 return false;
538 uses = new std::list<TexUse>[texes.size()];
539 if (!uses)
540 return false;
541 for (size_t i = 0; i < texes.size(); ++i) {
542 findFirstUses(texes[i], uses[i]);
543 }
544
545 // determine the barrier level at each use
546 for (size_t i = 0; i < texes.size(); ++i) {
547 for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end();
548 ++u) {
549 BasicBlock *tb = texes[i]->bb;
550 BasicBlock *ub = u->insn->bb;
551 if (tb == ub) {
552 u->level = 0;
553 for (size_t j = i + 1; j < texes.size() &&
554 texes[j]->bb == tb && texes[j]->serial < u->insn->serial;
555 ++j)
556 u->level++;
557 } else {
558 u->level = fn->cfg.findLightestPathWeight(&tb->cfg,
559 &ub->cfg, texCounts);
560 if (u->level < 0) {
561 WARN("Failed to find path TEX -> TEXBAR\n");
562 u->level = 0;
563 continue;
564 }
565 // this counted all TEXes in the origin block, correct that
566 u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;
567 // and did not count the TEXes in the destination block, add those
568 for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&
569 texes[j]->bb == ub && texes[j]->serial < u->insn->serial;
570 ++j)
571 u->level++;
572 }
573 assert(u->level >= 0);
574 useVec.push_back(*u);
575 }
576 }
577 delete[] uses;
578
579 // insert the barriers
580 for (size_t i = 0; i < useVec.size(); ++i) {
581 Instruction *prev = useVec[i].insn->prev;
582 if (useVec[i].level < 0)
583 continue;
584 if (prev && prev->op == OP_TEXBAR) {
585 if (prev->subOp > useVec[i].level)
586 prev->subOp = useVec[i].level;
587 prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));
588 } else {
589 Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);
590 bar->fixed = 1;
591 bar->subOp = useVec[i].level;
592 // make use explicit to ease latency calculation
593 bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));
594 useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);
595 }
596 }
597
598 if (fn->getProgram()->optLevel < 3)
599 return true;
600
601 std::vector<Limits> limitT, limitB, limitS; // entry, exit, single
602
603 limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));
604 limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));
605 limitS.resize(fn->allBBlocks.getSize());
606
607 // cull unneeded barriers (should do that earlier, but for simplicity)
608 IteratorRef bi = fn->cfg.iteratorCFG();
609 // first calculate min/max outstanding TEXes for each BB
610 for (bi->reset(); !bi->end(); bi->next()) {
611 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
612 BasicBlock *bb = BasicBlock::get(n);
613 int min = 0;
614 int max = std::numeric_limits<int>::max();
615 for (Instruction *i = bb->getFirst(); i; i = i->next) {
616 if (isTextureOp(i->op)) {
617 min++;
618 if (max < std::numeric_limits<int>::max())
619 max++;
620 } else
621 if (i->op == OP_TEXBAR) {
622 min = MIN2(min, i->subOp);
623 max = MIN2(max, i->subOp);
624 }
625 }
626 // limits when looking at an isolated block
627 limitS[bb->getId()].min = min;
628 limitS[bb->getId()].max = max;
629 }
630 // propagate the min/max values
631 for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {
632 for (bi->reset(); !bi->end(); bi->next()) {
633 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
634 BasicBlock *bb = BasicBlock::get(n);
635 const int bbId = bb->getId();
636 for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {
637 BasicBlock *in = BasicBlock::get(ei.getNode());
638 const int inId = in->getId();
639 limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);
640 limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);
641 }
642 // I just hope this is correct ...
643 if (limitS[bbId].max == std::numeric_limits<int>::max()) {
644 // no barrier
645 limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;
646 limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;
647 } else {
648 // block contained a barrier
649 limitB[bbId].min = MIN2(limitS[bbId].max,
650 limitT[bbId].min + limitS[bbId].min);
651 limitB[bbId].max = MIN2(limitS[bbId].max,
652 limitT[bbId].max + limitS[bbId].min);
653 }
654 }
655 }
656 // finally delete unnecessary barriers
657 for (bi->reset(); !bi->end(); bi->next()) {
658 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
659 BasicBlock *bb = BasicBlock::get(n);
660 Instruction *prev = NULL;
661 Instruction *next;
662 int max = limitT[bb->getId()].max;
663 for (Instruction *i = bb->getFirst(); i; i = next) {
664 next = i->next;
665 if (i->op == OP_TEXBAR) {
666 if (i->subOp >= max) {
667 delete_Instruction(prog, i);
668 i = NULL;
669 } else {
670 max = i->subOp;
671 if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {
672 delete_Instruction(prog, prev);
673 prev = NULL;
674 }
675 }
676 } else
677 if (isTextureOp(i->op)) {
678 max++;
679 }
680 if (i && !i->isNop())
681 prev = i;
682 }
683 }
684 return true;
685 }
686
687 bool
visit(Function * fn)688 NVC0LegalizePostRA::visit(Function *fn)
689 {
690 if (needTexBar)
691 insertTextureBarriers(fn);
692
693 rZero = new_LValue(fn, FILE_GPR);
694 pOne = new_LValue(fn, FILE_PREDICATE);
695 carry = new_LValue(fn, FILE_FLAGS);
696
697 rZero->reg.data.id = (prog->getTarget()->getChipset() >= NVISA_GK20A_CHIPSET) ? 255 : 63;
698 carry->reg.data.id = 0;
699 pOne->reg.data.id = 7;
700
701 return true;
702 }
703
704 void
replaceZero(Instruction * i)705 NVC0LegalizePostRA::replaceZero(Instruction *i)
706 {
707 for (int s = 0; i->srcExists(s); ++s) {
708 if (s == 2 && i->op == OP_SUCLAMP)
709 continue;
710 if (s == 1 && i->op == OP_SHLADD)
711 continue;
712 ImmediateValue *imm = i->getSrc(s)->asImm();
713 if (imm) {
714 if (i->op == OP_SELP && s == 2) {
715 i->setSrc(s, pOne);
716 if (imm->reg.data.u64 == 0)
717 i->src(s).mod = i->src(s).mod ^ Modifier(NV50_IR_MOD_NOT);
718 } else if (imm->reg.data.u64 == 0) {
719 i->setSrc(s, rZero);
720 }
721 }
722 }
723 }
724
725 // replace CONT with BRA for single unconditional continue
726 bool
tryReplaceContWithBra(BasicBlock * bb)727 NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)
728 {
729 if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)
730 return false;
731 Graph::EdgeIterator ei = bb->cfg.incident();
732 if (ei.getType() != Graph::Edge::BACK)
733 ei.next();
734 if (ei.getType() != Graph::Edge::BACK)
735 return false;
736 BasicBlock *contBB = BasicBlock::get(ei.getNode());
737
738 if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||
739 contBB->getExit()->getPredicate())
740 return false;
741 contBB->getExit()->op = OP_BRA;
742 bb->remove(bb->getEntry()); // delete PRECONT
743
744 ei.next();
745 assert(ei.end() || ei.getType() != Graph::Edge::BACK);
746 return true;
747 }
748
749 // replace branches to join blocks with join ops
750 void
propagateJoin(BasicBlock * bb)751 NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)
752 {
753 if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)
754 return;
755 for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
756 BasicBlock *in = BasicBlock::get(ei.getNode());
757 Instruction *exit = in->getExit();
758 if (!exit) {
759 in->insertTail(new FlowInstruction(func, OP_JOIN, bb));
760 // there should always be a terminator instruction
761 WARN("inserted missing terminator in BB:%i\n", in->getId());
762 } else
763 if (exit->op == OP_BRA) {
764 exit->op = OP_JOIN;
765 exit->asFlow()->limit = 1; // must-not-propagate marker
766 }
767 }
768 bb->remove(bb->getEntry());
769 }
770
771 // replaces instructions which would end up as f2f or i2i with faster
772 // alternatives:
773 // - fabs(a) -> fadd(0, abs a)
774 // - fneg(a) -> fadd(neg 0, neg a)
775 // - ineg(a) -> iadd(0, neg a)
776 // - fneg(abs a) -> fadd(neg 0, neg abs a)
777 // - sat(a) -> sat add(0, a)
778 void
replaceCvt(Instruction * cvt)779 NVC0LegalizePostRA::replaceCvt(Instruction *cvt)
780 {
781 if (!isFloatType(cvt->sType) && typeSizeof(cvt->sType) != 4)
782 return;
783 if (cvt->sType != cvt->dType)
784 return;
785 // we could make it work, but in this case we have optimizations disabled
786 // and we don't really care either way.
787 if (cvt->src(0).getFile() != FILE_GPR &&
788 cvt->src(0).getFile() != FILE_MEMORY_CONST)
789 return;
790
791 Modifier mod0, mod1;
792
793 switch (cvt->op) {
794 case OP_ABS:
795 if (cvt->src(0).mod)
796 return;
797 if (!isFloatType(cvt->sType))
798 return;
799 mod0 = 0;
800 mod1 = NV50_IR_MOD_ABS;
801 break;
802 case OP_NEG:
803 if (!isFloatType(cvt->sType) && cvt->src(0).mod)
804 return;
805 if (isFloatType(cvt->sType) &&
806 (cvt->src(0).mod && cvt->src(0).mod != Modifier(NV50_IR_MOD_ABS)))
807 return;
808
809 mod0 = isFloatType(cvt->sType) ? NV50_IR_MOD_NEG : 0;
810 mod1 = cvt->src(0).mod == Modifier(NV50_IR_MOD_ABS) ?
811 NV50_IR_MOD_NEG_ABS : NV50_IR_MOD_NEG;
812 break;
813 case OP_SAT:
814 if (!isFloatType(cvt->sType) && cvt->src(0).mod.abs())
815 return;
816 mod0 = 0;
817 mod1 = cvt->src(0).mod;
818 cvt->saturate = true;
819 break;
820 default:
821 return;
822 }
823
824 cvt->op = OP_ADD;
825 cvt->moveSources(0, 1);
826 cvt->setSrc(0, rZero);
827 cvt->src(0).mod = mod0;
828 cvt->src(1).mod = mod1;
829 }
830
831 bool
visit(BasicBlock * bb)832 NVC0LegalizePostRA::visit(BasicBlock *bb)
833 {
834 Instruction *i, *next;
835
836 // remove pseudo operations and non-fixed no-ops, split 64 bit operations
837 for (i = bb->getFirst(); i; i = next) {
838 next = i->next;
839 if (i->op == OP_EMIT || i->op == OP_RESTART) {
840 if (!i->getDef(0)->refCount())
841 i->setDef(0, NULL);
842 if (i->src(0).getFile() == FILE_IMMEDIATE)
843 i->setSrc(0, rZero); // initial value must be 0
844 replaceZero(i);
845 } else
846 if (i->isNop()) {
847 bb->remove(i);
848 } else
849 if (i->op == OP_BAR && i->subOp == NV50_IR_SUBOP_BAR_SYNC &&
850 prog->getType() != Program::TYPE_COMPUTE) {
851 // It seems like barriers are never required for tessellation since
852 // the warp size is 32, and there are always at most 32 tcs threads.
853 bb->remove(i);
854 } else
855 if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LDC_IS) {
856 int offset = i->src(0).get()->reg.data.offset;
857 if (abs(offset) >= 0x10000)
858 i->src(0).get()->reg.fileIndex += offset >> 16;
859 i->src(0).get()->reg.data.offset = (int)(short)offset;
860 } else {
861 // TODO: Move this to before register allocation for operations that
862 // need the $c register !
863 if (typeSizeof(i->sType) == 8 || typeSizeof(i->dType) == 8) {
864 Instruction *hi;
865 hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);
866 if (hi)
867 next = hi;
868 }
869
870 if (i->op != OP_MOV && i->op != OP_PFETCH)
871 replaceZero(i);
872
873 if (i->op == OP_SAT || i->op == OP_NEG || i->op == OP_ABS)
874 replaceCvt(i);
875 }
876 }
877 if (!bb->getEntry())
878 return true;
879
880 if (!tryReplaceContWithBra(bb))
881 propagateJoin(bb);
882
883 return true;
884 }
885
NVC0LoweringPass(Program * prog)886 NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())
887 {
888 bld.setProgram(prog);
889 }
890
891 bool
visit(Function * fn)892 NVC0LoweringPass::visit(Function *fn)
893 {
894 if (prog->getType() == Program::TYPE_GEOMETRY) {
895 assert(!strncmp(fn->getName(), "MAIN", 4));
896 // TODO: when we generate actual functions pass this value along somehow
897 bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);
898 gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();
899 if (fn->cfgExit) {
900 bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);
901 if (prog->getTarget()->getChipset() >= NVISA_GV100_CHIPSET)
902 bld.mkOp1(OP_FINAL, TYPE_NONE, NULL, gpEmitAddress)->fixed = 1;
903 bld.mkMovToReg(0, gpEmitAddress);
904 }
905 }
906 return true;
907 }
908
909 bool
visit(BasicBlock * bb)910 NVC0LoweringPass::visit(BasicBlock *bb)
911 {
912 return true;
913 }
914
915 inline Value *
loadTexHandle(Value * ptr,unsigned int slot)916 NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)
917 {
918 uint8_t b = prog->driver->io.auxCBSlot;
919 uint32_t off = prog->driver->io.texBindBase + slot * 4;
920
921 if (ptr)
922 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(2));
923
924 return bld.
925 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
926 }
927
928 // move array source to first slot, convert to u16, add indirections
929 bool
handleTEX(TexInstruction * i)930 NVC0LoweringPass::handleTEX(TexInstruction *i)
931 {
932 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
933 const int arg = i->tex.target.getArgCount();
934 const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);
935 const int chipset = prog->getTarget()->getChipset();
936
937 /* Only normalize in the non-explicit derivatives case. For explicit
938 * derivatives, this is handled in handleManualTXD.
939 */
940 if (i->tex.target.isCube() && i->dPdx[0].get() == NULL) {
941 Value *src[3], *val;
942 int c;
943 for (c = 0; c < 3; ++c)
944 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), i->getSrc(c));
945 val = bld.getScratch();
946 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
947 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
948 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
949 for (c = 0; c < 3; ++c) {
950 i->setSrc(c, bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(),
951 i->getSrc(c), val));
952 }
953 }
954
955 // Arguments to the TEX instruction are a little insane. Even though the
956 // encoding is identical between SM20 and SM30, the arguments mean
957 // different things between Fermi and Kepler+. A lot of arguments are
958 // optional based on flags passed to the instruction. This summarizes the
959 // order of things.
960 //
961 // Fermi:
962 // array/indirect
963 // coords
964 // sample
965 // lod bias
966 // depth compare
967 // offsets:
968 // - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)
969 // - other: 4 bits each, single reg
970 //
971 // Kepler+:
972 // indirect handle
973 // array (+ offsets for txd in upper 16 bits)
974 // coords
975 // sample
976 // lod bias
977 // depth compare
978 // offsets (same as fermi, except txd which takes it with array)
979 //
980 // Maxwell (tex):
981 // array
982 // coords
983 // indirect handle
984 // sample
985 // lod bias
986 // depth compare
987 // offsets
988 //
989 // Maxwell (txd):
990 // indirect handle
991 // coords
992 // array + offsets
993 // derivatives
994
995 if (chipset >= NVISA_GK104_CHIPSET) {
996 if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
997 // XXX this ignores tsc, and assumes a 1:1 mapping
998 assert(i->tex.rIndirectSrc >= 0);
999 if (!i->tex.bindless) {
1000 Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r);
1001 i->tex.r = 0xff;
1002 i->tex.s = 0x1f;
1003 i->setIndirectR(hnd);
1004 }
1005 i->setIndirectS(NULL);
1006 } else if (i->tex.r == i->tex.s || i->op == OP_TXF) {
1007 if (i->tex.r == 0xffff)
1008 i->tex.r = prog->driver->io.fbtexBindBase / 4;
1009 else
1010 i->tex.r += prog->driver->io.texBindBase / 4;
1011 i->tex.s = 0; // only a single cX[] value possible here
1012 } else {
1013 Value *hnd = bld.getScratch();
1014 Value *rHnd = loadTexHandle(NULL, i->tex.r);
1015 Value *sHnd = loadTexHandle(NULL, i->tex.s);
1016
1017 bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);
1018
1019 i->tex.r = 0; // not used for indirect tex
1020 i->tex.s = 0;
1021 i->setIndirectR(hnd);
1022 }
1023 if (i->tex.target.isArray()) {
1024 LValue *layer = new_LValue(func, FILE_GPR);
1025 Value *src = i->getSrc(lyr);
1026 const int sat = (i->op == OP_TXF) ? 1 : 0;
1027 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
1028 bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;
1029 if (i->op != OP_TXD || chipset < NVISA_GM107_CHIPSET) {
1030 for (int s = dim; s >= 1; --s)
1031 i->setSrc(s, i->getSrc(s - 1));
1032 i->setSrc(0, layer);
1033 } else {
1034 i->setSrc(dim, layer);
1035 }
1036 }
1037 // Move the indirect reference to the first place
1038 if (i->tex.rIndirectSrc >= 0 && (
1039 i->op == OP_TXD || chipset < NVISA_GM107_CHIPSET)) {
1040 Value *hnd = i->getIndirectR();
1041
1042 i->setIndirectR(NULL);
1043 i->moveSources(0, 1);
1044 i->setSrc(0, hnd);
1045 i->tex.rIndirectSrc = 0;
1046 i->tex.sIndirectSrc = -1;
1047 }
1048 // Move the indirect reference to right after the coords
1049 else if (i->tex.rIndirectSrc >= 0 && chipset >= NVISA_GM107_CHIPSET) {
1050 Value *hnd = i->getIndirectR();
1051
1052 i->setIndirectR(NULL);
1053 i->moveSources(arg, 1);
1054 i->setSrc(arg, hnd);
1055 i->tex.rIndirectSrc = 0;
1056 i->tex.sIndirectSrc = -1;
1057 }
1058 } else
1059 // (nvc0) generate and move the tsc/tic/array source to the front
1060 if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
1061 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
1062
1063 Value *ticRel = i->getIndirectR();
1064 Value *tscRel = i->getIndirectS();
1065
1066 if (i->tex.r == 0xffff) {
1067 i->tex.r = 0x20;
1068 i->tex.s = 0x10;
1069 }
1070
1071 if (ticRel) {
1072 i->setSrc(i->tex.rIndirectSrc, NULL);
1073 if (i->tex.r)
1074 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
1075 ticRel, bld.mkImm(i->tex.r));
1076 }
1077 if (tscRel) {
1078 i->setSrc(i->tex.sIndirectSrc, NULL);
1079 if (i->tex.s)
1080 tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
1081 tscRel, bld.mkImm(i->tex.s));
1082 }
1083
1084 Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;
1085 if (arrayIndex) {
1086 for (int s = dim; s >= 1; --s)
1087 i->setSrc(s, i->getSrc(s - 1));
1088 i->setSrc(0, arrayIndex);
1089 } else {
1090 i->moveSources(0, 1);
1091 }
1092
1093 if (arrayIndex) {
1094 int sat = (i->op == OP_TXF) ? 1 : 0;
1095 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
1096 bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;
1097 } else {
1098 bld.loadImm(src, 0);
1099 }
1100
1101 if (ticRel)
1102 bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);
1103 if (tscRel)
1104 bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);
1105
1106 i->setSrc(0, src);
1107 }
1108
1109 // For nvc0, the sample id has to be in the second operand, as the offset
1110 // does. Right now we don't know how to pass both in, and this case can't
1111 // happen with OpenGL. On nve0, the sample id is part of the texture
1112 // coordinate argument.
1113 assert(chipset >= NVISA_GK104_CHIPSET ||
1114 !i->tex.useOffsets || !i->tex.target.isMS());
1115
1116 // offset is between lod and dc
1117 if (i->tex.useOffsets) {
1118 int n, c;
1119 int s = i->srcCount(0xff, true);
1120 if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {
1121 if (i->tex.target.isShadow())
1122 s--;
1123 if (i->srcExists(s)) // move potential predicate out of the way
1124 i->moveSources(s, 1);
1125 if (i->tex.useOffsets == 4 && i->srcExists(s + 1))
1126 i->moveSources(s + 1, 1);
1127 }
1128 if (i->op == OP_TXG) {
1129 // Either there is 1 offset, which goes into the 2 low bytes of the
1130 // first source, or there are 4 offsets, which go into 2 sources (8
1131 // values, 1 byte each).
1132 Value *offs[2] = {NULL, NULL};
1133 for (n = 0; n < i->tex.useOffsets; n++) {
1134 for (c = 0; c < 2; ++c) {
1135 if ((n % 2) == 0 && c == 0)
1136 bld.mkMov(offs[n / 2] = bld.getScratch(), i->offset[n][c].get());
1137 else
1138 bld.mkOp3(OP_INSBF, TYPE_U32,
1139 offs[n / 2],
1140 i->offset[n][c].get(),
1141 bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),
1142 offs[n / 2]);
1143 }
1144 }
1145 i->setSrc(s, offs[0]);
1146 if (offs[1])
1147 i->setSrc(s + 1, offs[1]);
1148 } else {
1149 unsigned imm = 0;
1150 assert(i->tex.useOffsets == 1);
1151 for (c = 0; c < 3; ++c) {
1152 ImmediateValue val;
1153 if (!i->offset[0][c].getImmediate(val))
1154 assert(!"non-immediate offset passed to non-TXG");
1155 imm |= (val.reg.data.u32 & 0xf) << (c * 4);
1156 }
1157 if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {
1158 // The offset goes into the upper 16 bits of the array index. So
1159 // create it if it's not already there, and INSBF it if it already
1160 // is.
1161 s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;
1162 if (chipset >= NVISA_GM107_CHIPSET)
1163 s += dim;
1164 if (i->tex.target.isArray()) {
1165 Value *offset = bld.getScratch();
1166 bld.mkOp3(OP_INSBF, TYPE_U32, offset,
1167 bld.loadImm(NULL, imm), bld.mkImm(0xc10),
1168 i->getSrc(s));
1169 i->setSrc(s, offset);
1170 } else {
1171 i->moveSources(s, 1);
1172 i->setSrc(s, bld.loadImm(NULL, imm << 16));
1173 }
1174 } else {
1175 i->setSrc(s, bld.loadImm(NULL, imm));
1176 }
1177 }
1178 }
1179
1180 return true;
1181 }
1182
1183 bool
handleManualTXD(TexInstruction * i)1184 NVC0LoweringPass::handleManualTXD(TexInstruction *i)
1185 {
1186 // Always done from the l0 perspective. This is the way that NVIDIA's
1187 // driver does it, and doing it from the "current" lane's perpsective
1188 // doesn't seem to always work for reasons that aren't altogether clear,
1189 // even in frag shaders.
1190 //
1191 // Note that we must move not only the coordinates into lane0, but also all
1192 // ancillary arguments, like array indices and depth compare as they may
1193 // differ between lanes. Offsets for TXD are supposed to be uniform, so we
1194 // leave them alone.
1195 static const uint8_t qOps[2] =
1196 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) };
1197
1198 Value *def[4][4];
1199 Value *crd[3], *arr[2], *shadow;
1200 Instruction *tex;
1201 Value *zero = bld.loadImm(bld.getSSA(), 0);
1202 int l, c;
1203 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
1204
1205 // This function is invoked after handleTEX lowering, so we have to expect
1206 // the arguments in the order that the hw wants them. For Fermi, array and
1207 // indirect are both in the leading arg, while for Kepler, array and
1208 // indirect are separate (and both precede the coordinates). Maxwell is
1209 // handled in a separate function.
1210 int array;
1211 if (targ->getChipset() < NVISA_GK104_CHIPSET)
1212 array = i->tex.target.isArray() || i->tex.rIndirectSrc >= 0;
1213 else
1214 array = i->tex.target.isArray() + (i->tex.rIndirectSrc >= 0);
1215
1216 i->op = OP_TEX; // no need to clone dPdx/dPdy later
1217
1218 for (c = 0; c < dim; ++c)
1219 crd[c] = bld.getScratch();
1220 for (c = 0; c < array; ++c)
1221 arr[c] = bld.getScratch();
1222 shadow = bld.getScratch();
1223
1224 for (l = 0; l < 4; ++l) {
1225 Value *src[3], *val;
1226
1227 bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
1228 // we're using the texture result from lane 0 in all cases, so make sure
1229 // that lane 0 is pointing at the proper array index, indirect value,
1230 // and depth compare.
1231 if (l != 0) {
1232 for (c = 0; c < array; ++c)
1233 bld.mkQuadop(0x00, arr[c], l, i->getSrc(c), zero);
1234 if (i->tex.target.isShadow()) {
1235 // The next argument after coords is the depth compare
1236 bld.mkQuadop(0x00, shadow, l, i->getSrc(array + dim), zero);
1237 }
1238 }
1239 // mov position coordinates from lane l to all lanes
1240 for (c = 0; c < dim; ++c)
1241 bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);
1242 // add dPdx from lane l to lanes dx
1243 for (c = 0; c < dim; ++c)
1244 bld.mkQuadop(qOps[0], crd[c], l, i->dPdx[c].get(), crd[c]);
1245 // add dPdy from lane l to lanes dy
1246 for (c = 0; c < dim; ++c)
1247 bld.mkQuadop(qOps[1], crd[c], l, i->dPdy[c].get(), crd[c]);
1248 // normalize cube coordinates
1249 if (i->tex.target.isCube()) {
1250 for (c = 0; c < 3; ++c)
1251 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), crd[c]);
1252 val = bld.getScratch();
1253 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
1254 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
1255 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
1256 for (c = 0; c < 3; ++c)
1257 src[c] = bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(), crd[c], val);
1258 } else {
1259 for (c = 0; c < dim; ++c)
1260 src[c] = crd[c];
1261 }
1262 // texture
1263 bld.insert(tex = cloneForward(func, i));
1264 if (l != 0) {
1265 for (c = 0; c < array; ++c)
1266 tex->setSrc(c, arr[c]);
1267 if (i->tex.target.isShadow())
1268 tex->setSrc(array + dim, shadow);
1269 }
1270 for (c = 0; c < dim; ++c)
1271 tex->setSrc(c + array, src[c]);
1272 // broadcast results from lane 0 to all lanes so that the moves *into*
1273 // the target lane pick up the proper value.
1274 if (l != 0)
1275 for (c = 0; i->defExists(c); ++c)
1276 bld.mkQuadop(0x00, tex->getDef(c), 0, tex->getDef(c), zero);
1277 bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
1278
1279 // save results
1280 for (c = 0; i->defExists(c); ++c) {
1281 Instruction *mov;
1282 def[c][l] = bld.getSSA();
1283 mov = bld.mkMov(def[c][l], tex->getDef(c));
1284 mov->fixed = 1;
1285 mov->lanes = 1 << l;
1286 }
1287 }
1288
1289 for (c = 0; i->defExists(c); ++c) {
1290 Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
1291 for (l = 0; l < 4; ++l)
1292 u->setSrc(l, def[c][l]);
1293 }
1294
1295 i->bb->remove(i);
1296 return true;
1297 }
1298
1299 bool
handleTXD(TexInstruction * txd)1300 NVC0LoweringPass::handleTXD(TexInstruction *txd)
1301 {
1302 int dim = txd->tex.target.getDim() + txd->tex.target.isCube();
1303 unsigned arg = txd->tex.target.getArgCount();
1304 unsigned expected_args = arg;
1305 const int chipset = prog->getTarget()->getChipset();
1306
1307 if (chipset >= NVISA_GK104_CHIPSET) {
1308 if (!txd->tex.target.isArray() && txd->tex.useOffsets)
1309 expected_args++;
1310 if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)
1311 expected_args++;
1312 } else {
1313 if (txd->tex.useOffsets)
1314 expected_args++;
1315 if (!txd->tex.target.isArray() && (
1316 txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))
1317 expected_args++;
1318 }
1319
1320 if (expected_args > 4 ||
1321 dim > 2 ||
1322 txd->tex.target.isShadow())
1323 txd->op = OP_TEX;
1324
1325 handleTEX(txd);
1326 while (txd->srcExists(arg))
1327 ++arg;
1328
1329 txd->tex.derivAll = true;
1330 if (txd->op == OP_TEX)
1331 return handleManualTXD(txd);
1332
1333 assert(arg == expected_args);
1334 for (int c = 0; c < dim; ++c) {
1335 txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);
1336 txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);
1337 txd->dPdx[c].set(NULL);
1338 txd->dPdy[c].set(NULL);
1339 }
1340
1341 // In this case we have fewer than 4 "real" arguments, which means that
1342 // handleTEX didn't apply any padding. However we have to make sure that
1343 // the second "group" of arguments still gets padded up to 4.
1344 if (chipset >= NVISA_GK104_CHIPSET) {
1345 int s = arg + 2 * dim;
1346 if (s >= 4 && s < 7) {
1347 if (txd->srcExists(s)) // move potential predicate out of the way
1348 txd->moveSources(s, 7 - s);
1349 while (s < 7)
1350 txd->setSrc(s++, bld.loadImm(NULL, 0));
1351 }
1352 }
1353
1354 return true;
1355 }
1356
1357 bool
handleTXQ(TexInstruction * txq)1358 NVC0LoweringPass::handleTXQ(TexInstruction *txq)
1359 {
1360 const int chipset = prog->getTarget()->getChipset();
1361 if (chipset >= NVISA_GK104_CHIPSET && txq->tex.rIndirectSrc < 0)
1362 txq->tex.r += prog->driver->io.texBindBase / 4;
1363
1364 if (txq->tex.rIndirectSrc < 0)
1365 return true;
1366
1367 Value *ticRel = txq->getIndirectR();
1368
1369 txq->setIndirectS(NULL);
1370 txq->tex.sIndirectSrc = -1;
1371
1372 assert(ticRel);
1373
1374 if (chipset < NVISA_GK104_CHIPSET) {
1375 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
1376
1377 txq->setSrc(txq->tex.rIndirectSrc, NULL);
1378 if (txq->tex.r)
1379 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
1380 ticRel, bld.mkImm(txq->tex.r));
1381
1382 bld.mkOp2(OP_SHL, TYPE_U32, src, ticRel, bld.mkImm(0x17));
1383
1384 txq->moveSources(0, 1);
1385 txq->setSrc(0, src);
1386 } else {
1387 Value *hnd = loadTexHandle(txq->getIndirectR(), txq->tex.r);
1388 txq->tex.r = 0xff;
1389 txq->tex.s = 0x1f;
1390
1391 txq->setIndirectR(NULL);
1392 txq->moveSources(0, 1);
1393 txq->setSrc(0, hnd);
1394 txq->tex.rIndirectSrc = 0;
1395 }
1396
1397 return true;
1398 }
1399
1400 bool
handleTXLQ(TexInstruction * i)1401 NVC0LoweringPass::handleTXLQ(TexInstruction *i)
1402 {
1403 /* The outputs are inverted compared to what the TGSI instruction
1404 * expects. Take that into account in the mask.
1405 */
1406 assert((i->tex.mask & ~3) == 0);
1407 if (i->tex.mask == 1)
1408 i->tex.mask = 2;
1409 else if (i->tex.mask == 2)
1410 i->tex.mask = 1;
1411 handleTEX(i);
1412 bld.setPosition(i, true);
1413
1414 /* The returned values are not quite what we want:
1415 * (a) convert from s16/u16 to f32
1416 * (b) multiply by 1/256
1417 */
1418 for (int def = 0; def < 2; ++def) {
1419 if (!i->defExists(def))
1420 continue;
1421 enum DataType type = TYPE_S16;
1422 if (i->tex.mask == 2 || def > 0)
1423 type = TYPE_U16;
1424 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));
1425 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),
1426 i->getDef(def), bld.loadImm(NULL, 1.0f / 256));
1427 }
1428 if (i->tex.mask == 3) {
1429 LValue *t = new_LValue(func, FILE_GPR);
1430 bld.mkMov(t, i->getDef(0));
1431 bld.mkMov(i->getDef(0), i->getDef(1));
1432 bld.mkMov(i->getDef(1), t);
1433 }
1434 return true;
1435 }
1436
1437 bool
handleBUFQ(Instruction * bufq)1438 NVC0LoweringPass::handleBUFQ(Instruction *bufq)
1439 {
1440 bufq->op = OP_MOV;
1441 bufq->setSrc(0, loadBufLength32(bufq->getIndirect(0, 1),
1442 bufq->getSrc(0)->reg.fileIndex * 16));
1443 bufq->setIndirect(0, 0, NULL);
1444 bufq->setIndirect(0, 1, NULL);
1445 return true;
1446 }
1447
1448 void
handleSharedATOMNVE4(Instruction * atom)1449 NVC0LoweringPass::handleSharedATOMNVE4(Instruction *atom)
1450 {
1451 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1452
1453 BasicBlock *currBB = atom->bb;
1454 BasicBlock *tryLockBB = atom->bb->splitBefore(atom, false);
1455 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1456 BasicBlock *setAndUnlockBB = new BasicBlock(func);
1457 BasicBlock *failLockBB = new BasicBlock(func);
1458
1459 bld.setPosition(currBB, true);
1460 assert(!currBB->joinAt);
1461 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1462
1463 CmpInstruction *pred =
1464 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1465 TYPE_U32, bld.mkImm(0), bld.mkImm(1));
1466
1467 bld.mkFlow(OP_BRA, tryLockBB, CC_ALWAYS, NULL);
1468 currBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::TREE);
1469
1470 bld.setPosition(tryLockBB, true);
1471
1472 Instruction *ld =
1473 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1474 atom->getIndirect(0, 0));
1475 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1476 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1477
1478 bld.mkFlow(OP_BRA, setAndUnlockBB, CC_P, ld->getDef(1));
1479 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1480 tryLockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::CROSS);
1481 tryLockBB->cfg.attach(&setAndUnlockBB->cfg, Graph::Edge::TREE);
1482
1483 tryLockBB->cfg.detach(&joinBB->cfg);
1484 bld.remove(atom);
1485
1486 bld.setPosition(setAndUnlockBB, true);
1487 Value *stVal;
1488 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1489 // Read the old value, and write the new one.
1490 stVal = atom->getSrc(1);
1491 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1492 CmpInstruction *set =
1493 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(),
1494 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1495
1496 bld.mkCmp(OP_SLCT, CC_NE, TYPE_U32, (stVal = bld.getSSA()),
1497 TYPE_U32, atom->getSrc(2), ld->getDef(0), set->getDef(0));
1498 } else {
1499 operation op;
1500
1501 switch (atom->subOp) {
1502 case NV50_IR_SUBOP_ATOM_ADD:
1503 op = OP_ADD;
1504 break;
1505 case NV50_IR_SUBOP_ATOM_AND:
1506 op = OP_AND;
1507 break;
1508 case NV50_IR_SUBOP_ATOM_OR:
1509 op = OP_OR;
1510 break;
1511 case NV50_IR_SUBOP_ATOM_XOR:
1512 op = OP_XOR;
1513 break;
1514 case NV50_IR_SUBOP_ATOM_MIN:
1515 op = OP_MIN;
1516 break;
1517 case NV50_IR_SUBOP_ATOM_MAX:
1518 op = OP_MAX;
1519 break;
1520 default:
1521 assert(0);
1522 return;
1523 }
1524
1525 stVal = bld.mkOp2v(op, atom->dType, bld.getSSA(), ld->getDef(0),
1526 atom->getSrc(1));
1527 }
1528
1529 Instruction *st =
1530 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1531 atom->getIndirect(0, 0), stVal);
1532 st->setDef(0, pred->getDef(0));
1533 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1534
1535 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1536 setAndUnlockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::TREE);
1537
1538 // Lock until the store has not been performed.
1539 bld.setPosition(failLockBB, true);
1540 bld.mkFlow(OP_BRA, tryLockBB, CC_NOT_P, pred->getDef(0));
1541 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1542 failLockBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::BACK);
1543 failLockBB->cfg.attach(&joinBB->cfg, Graph::Edge::TREE);
1544
1545 bld.setPosition(joinBB, false);
1546 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1547 }
1548
1549 void
handleSharedATOM(Instruction * atom)1550 NVC0LoweringPass::handleSharedATOM(Instruction *atom)
1551 {
1552 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1553
1554 BasicBlock *currBB = atom->bb;
1555 BasicBlock *tryLockAndSetBB = atom->bb->splitBefore(atom, false);
1556 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1557
1558 bld.setPosition(currBB, true);
1559 assert(!currBB->joinAt);
1560 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1561
1562 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_ALWAYS, NULL);
1563 currBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::TREE);
1564
1565 bld.setPosition(tryLockAndSetBB, true);
1566
1567 Instruction *ld =
1568 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1569 atom->getIndirect(0, 0));
1570 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1571 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1572
1573 Value *stVal;
1574 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1575 // Read the old value, and write the new one.
1576 stVal = atom->getSrc(1);
1577 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1578 CmpInstruction *set =
1579 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1580 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1581 set->setPredicate(CC_P, ld->getDef(1));
1582
1583 Instruction *selp =
1584 bld.mkOp3(OP_SELP, TYPE_U32, bld.getSSA(), ld->getDef(0),
1585 atom->getSrc(2), set->getDef(0));
1586 selp->src(2).mod = Modifier(NV50_IR_MOD_NOT);
1587 selp->setPredicate(CC_P, ld->getDef(1));
1588
1589 stVal = selp->getDef(0);
1590 } else {
1591 operation op;
1592
1593 switch (atom->subOp) {
1594 case NV50_IR_SUBOP_ATOM_ADD:
1595 op = OP_ADD;
1596 break;
1597 case NV50_IR_SUBOP_ATOM_AND:
1598 op = OP_AND;
1599 break;
1600 case NV50_IR_SUBOP_ATOM_OR:
1601 op = OP_OR;
1602 break;
1603 case NV50_IR_SUBOP_ATOM_XOR:
1604 op = OP_XOR;
1605 break;
1606 case NV50_IR_SUBOP_ATOM_MIN:
1607 op = OP_MIN;
1608 break;
1609 case NV50_IR_SUBOP_ATOM_MAX:
1610 op = OP_MAX;
1611 break;
1612 default:
1613 assert(0);
1614 return;
1615 }
1616
1617 Instruction *i =
1618 bld.mkOp2(op, atom->dType, bld.getSSA(), ld->getDef(0),
1619 atom->getSrc(1));
1620 i->setPredicate(CC_P, ld->getDef(1));
1621
1622 stVal = i->getDef(0);
1623 }
1624
1625 Instruction *st =
1626 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1627 atom->getIndirect(0, 0), stVal);
1628 st->setPredicate(CC_P, ld->getDef(1));
1629 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1630
1631 // Loop until the lock is acquired.
1632 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_NOT_P, ld->getDef(1));
1633 tryLockAndSetBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::BACK);
1634 tryLockAndSetBB->cfg.attach(&joinBB->cfg, Graph::Edge::CROSS);
1635 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1636
1637 bld.remove(atom);
1638
1639 bld.setPosition(joinBB, false);
1640 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1641 }
1642
1643 bool
handleATOM(Instruction * atom)1644 NVC0LoweringPass::handleATOM(Instruction *atom)
1645 {
1646 SVSemantic sv;
1647 Value *ptr = atom->getIndirect(0, 0), *ind = atom->getIndirect(0, 1), *base;
1648
1649 switch (atom->src(0).getFile()) {
1650 case FILE_MEMORY_LOCAL:
1651 sv = SV_LBASE;
1652 break;
1653 case FILE_MEMORY_SHARED:
1654 // For Fermi/Kepler, we have to use ld lock/st unlock to perform atomic
1655 // operations on shared memory. For Maxwell, ATOMS is enough.
1656 if (targ->getChipset() < NVISA_GK104_CHIPSET)
1657 handleSharedATOM(atom);
1658 else if (targ->getChipset() < NVISA_GM107_CHIPSET)
1659 handleSharedATOMNVE4(atom);
1660 return true;
1661 case FILE_MEMORY_GLOBAL:
1662 return true;
1663 default:
1664 assert(atom->src(0).getFile() == FILE_MEMORY_BUFFER);
1665 base = loadBufInfo64(ind, atom->getSrc(0)->reg.fileIndex * 16);
1666 assert(base->reg.size == 8);
1667 if (ptr)
1668 base = bld.mkOp2v(OP_ADD, TYPE_U64, base, base, ptr);
1669 assert(base->reg.size == 8);
1670 atom->setIndirect(0, 0, base);
1671 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1672
1673 // Harden against out-of-bounds accesses
1674 Value *offset = bld.loadImm(NULL, atom->getSrc(0)->reg.data.offset + typeSizeof(atom->sType));
1675 Value *length = loadBufLength32(ind, atom->getSrc(0)->reg.fileIndex * 16);
1676 Value *pred = new_LValue(func, FILE_PREDICATE);
1677 if (ptr)
1678 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, ptr);
1679 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
1680 atom->setPredicate(CC_NOT_P, pred);
1681 if (atom->defExists(0)) {
1682 Value *zero, *dst = atom->getDef(0);
1683 atom->setDef(0, bld.getSSA());
1684
1685 bld.setPosition(atom, true);
1686 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
1687 ->setPredicate(CC_P, pred);
1688 bld.mkOp2(OP_UNION, TYPE_U32, dst, atom->getDef(0), zero);
1689 }
1690
1691 return true;
1692 }
1693 base =
1694 bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));
1695
1696 atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));
1697 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1698 if (ptr)
1699 base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);
1700 atom->setIndirect(0, 1, NULL);
1701 atom->setIndirect(0, 0, base);
1702
1703 return true;
1704 }
1705
1706 bool
handleCasExch(Instruction * cas,bool needCctl)1707 NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)
1708 {
1709 if (targ->getChipset() < NVISA_GM107_CHIPSET) {
1710 if (cas->src(0).getFile() == FILE_MEMORY_SHARED) {
1711 // ATOM_CAS and ATOM_EXCH are handled in handleSharedATOM().
1712 return false;
1713 }
1714 }
1715
1716 if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&
1717 cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)
1718 return false;
1719 bld.setPosition(cas, true);
1720
1721 if (needCctl) {
1722 Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));
1723 cctl->setIndirect(0, 0, cas->getIndirect(0, 0));
1724 cctl->fixed = 1;
1725 cctl->subOp = NV50_IR_SUBOP_CCTL_IV;
1726 if (cas->isPredicated())
1727 cctl->setPredicate(cas->cc, cas->getPredicate());
1728 }
1729
1730 if (cas->subOp == NV50_IR_SUBOP_ATOM_CAS &&
1731 targ->getChipset() < NVISA_GV100_CHIPSET) {
1732 // CAS is crazy. It's 2nd source is a double reg, and the 3rd source
1733 // should be set to the high part of the double reg or bad things will
1734 // happen elsewhere in the universe.
1735 // Also, it sometimes returns the new value instead of the old one
1736 // under mysterious circumstances.
1737 DataType ty = typeOfSize(typeSizeof(cas->dType) * 2);
1738 Value *dreg = bld.getSSA(typeSizeof(ty));
1739 bld.setPosition(cas, false);
1740 bld.mkOp2(OP_MERGE, ty, dreg, cas->getSrc(1), cas->getSrc(2));
1741 cas->setSrc(1, dreg);
1742 cas->setSrc(2, dreg);
1743 }
1744
1745 return true;
1746 }
1747
1748 inline Value *
loadResInfo32(Value * ptr,uint32_t off,uint16_t base)1749 NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off, uint16_t base)
1750 {
1751 uint8_t b = prog->driver->io.auxCBSlot;
1752 off += base;
1753
1754 return bld.
1755 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1756 }
1757
1758 inline Value *
loadResInfo64(Value * ptr,uint32_t off,uint16_t base)1759 NVC0LoweringPass::loadResInfo64(Value *ptr, uint32_t off, uint16_t base)
1760 {
1761 uint8_t b = prog->driver->io.auxCBSlot;
1762 off += base;
1763
1764 if (ptr)
1765 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1766
1767 return bld.
1768 mkLoadv(TYPE_U64, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off), ptr);
1769 }
1770
1771 inline Value *
loadResLength32(Value * ptr,uint32_t off,uint16_t base)1772 NVC0LoweringPass::loadResLength32(Value *ptr, uint32_t off, uint16_t base)
1773 {
1774 uint8_t b = prog->driver->io.auxCBSlot;
1775 off += base;
1776
1777 if (ptr)
1778 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1779
1780 return bld.
1781 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off + 8), ptr);
1782 }
1783
1784 inline Value *
loadBufInfo64(Value * ptr,uint32_t off)1785 NVC0LoweringPass::loadBufInfo64(Value *ptr, uint32_t off)
1786 {
1787 return loadResInfo64(ptr, off, prog->driver->io.bufInfoBase);
1788 }
1789
1790 inline Value *
loadBufLength32(Value * ptr,uint32_t off)1791 NVC0LoweringPass::loadBufLength32(Value *ptr, uint32_t off)
1792 {
1793 return loadResLength32(ptr, off, prog->driver->io.bufInfoBase);
1794 }
1795
1796 inline Value *
loadUboInfo64(Value * ptr,uint32_t off)1797 NVC0LoweringPass::loadUboInfo64(Value *ptr, uint32_t off)
1798 {
1799 return loadResInfo64(ptr, off, prog->driver->io.uboInfoBase);
1800 }
1801
1802 inline Value *
loadUboLength32(Value * ptr,uint32_t off)1803 NVC0LoweringPass::loadUboLength32(Value *ptr, uint32_t off)
1804 {
1805 return loadResLength32(ptr, off, prog->driver->io.uboInfoBase);
1806 }
1807
1808 inline Value *
loadMsInfo32(Value * ptr,uint32_t off)1809 NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)
1810 {
1811 uint8_t b = prog->driver->io.msInfoCBSlot;
1812 off += prog->driver->io.msInfoBase;
1813 return bld.
1814 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1815 }
1816
1817 inline Value *
loadSuInfo32(Value * ptr,int slot,uint32_t off,bool bindless)1818 NVC0LoweringPass::loadSuInfo32(Value *ptr, int slot, uint32_t off, bool bindless)
1819 {
1820 uint32_t base = slot * NVC0_SU_INFO__STRIDE;
1821
1822 // We don't upload surface info for bindless for GM107+
1823 assert(!bindless || targ->getChipset() < NVISA_GM107_CHIPSET);
1824
1825 if (ptr) {
1826 ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(slot));
1827 if (bindless)
1828 ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(511));
1829 else
1830 ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7));
1831 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(6));
1832 base = 0;
1833 }
1834 off += base;
1835
1836 return loadResInfo32(ptr, off, bindless ? prog->driver->io.bindlessBase :
1837 prog->driver->io.suInfoBase);
1838 }
1839
1840 Value *
loadMsAdjInfo32(TexInstruction::Target target,uint32_t index,int slot,Value * ind,bool bindless)1841 NVC0LoweringPass::loadMsAdjInfo32(TexInstruction::Target target, uint32_t index, int slot, Value *ind, bool bindless)
1842 {
1843 if (!bindless || targ->getChipset() < NVISA_GM107_CHIPSET)
1844 return loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(index), bindless);
1845
1846 assert(bindless);
1847
1848 Value *samples = bld.getSSA();
1849 // this shouldn't be lowered because it's being inserted before the current instruction
1850 TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1851 tex->tex.target = target;
1852 tex->tex.query = TXQ_TYPE;
1853 tex->tex.mask = 0x4;
1854 tex->tex.r = 0xff;
1855 tex->tex.s = 0x1f;
1856 tex->tex.rIndirectSrc = 0;
1857 tex->setDef(0, samples);
1858 tex->setSrc(0, ind);
1859 tex->setSrc(1, bld.loadImm(NULL, 0));
1860 bld.insert(tex);
1861
1862 // doesn't work with sample counts other than 1/2/4/8 but they aren't supported
1863 switch (index) {
1864 case 0: {
1865 Value *tmp = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), samples, bld.mkImm(2));
1866 return bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), tmp, bld.mkImm(2));
1867 }
1868 case 1: {
1869 Value *tmp = bld.mkCmp(OP_SET, CC_GT, TYPE_U32, bld.getSSA(), TYPE_U32, samples, bld.mkImm(2))->getDef(0);
1870 return bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), tmp, bld.mkImm(1));
1871 }
1872 default: {
1873 assert(false);
1874 return NULL;
1875 }
1876 }
1877 }
1878
getSuClampSubOp(const TexInstruction * su,int c)1879 static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)
1880 {
1881 switch (su->tex.target.getEnum()) {
1882 case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);
1883 case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1884 case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1885 case TEX_TARGET_1D_ARRAY: return (c == 1) ?
1886 NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :
1887 NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1888 case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1889 case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1890 case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1891 case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1892 case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1893 case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1894 case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1895 default:
1896 assert(0);
1897 return 0;
1898 }
1899 }
1900
1901 bool
handleSUQ(TexInstruction * suq)1902 NVC0LoweringPass::handleSUQ(TexInstruction *suq)
1903 {
1904 int mask = suq->tex.mask;
1905 int dim = suq->tex.target.getDim();
1906 int arg = dim + (suq->tex.target.isArray() || suq->tex.target.isCube());
1907 Value *ind = suq->getIndirectR();
1908 int slot = suq->tex.r;
1909 int c, d;
1910
1911 for (c = 0, d = 0; c < 3; ++c, mask >>= 1) {
1912 if (c >= arg || !(mask & 1))
1913 continue;
1914
1915 int offset;
1916
1917 if (c == 1 && suq->tex.target == TEX_TARGET_1D_ARRAY) {
1918 offset = NVC0_SU_INFO_SIZE(2);
1919 } else {
1920 offset = NVC0_SU_INFO_SIZE(c);
1921 }
1922 bld.mkMov(suq->getDef(d++), loadSuInfo32(ind, slot, offset, suq->tex.bindless));
1923 if (c == 2 && suq->tex.target.isCube())
1924 bld.mkOp2(OP_DIV, TYPE_U32, suq->getDef(d - 1), suq->getDef(d - 1),
1925 bld.loadImm(NULL, 6));
1926 }
1927
1928 if (mask & 1) {
1929 if (suq->tex.target.isMS()) {
1930 Value *ms_x = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(0), suq->tex.bindless);
1931 Value *ms_y = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(1), suq->tex.bindless);
1932 Value *ms = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), ms_x, ms_y);
1933 bld.mkOp2(OP_SHL, TYPE_U32, suq->getDef(d++), bld.loadImm(NULL, 1), ms);
1934 } else {
1935 bld.mkMov(suq->getDef(d++), bld.loadImm(NULL, 1));
1936 }
1937 }
1938
1939 bld.remove(suq);
1940 return true;
1941 }
1942
1943 void
adjustCoordinatesMS(TexInstruction * tex)1944 NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex)
1945 {
1946 const int arg = tex->tex.target.getArgCount();
1947 int slot = tex->tex.r;
1948
1949 if (tex->tex.target == TEX_TARGET_2D_MS)
1950 tex->tex.target = TEX_TARGET_2D;
1951 else
1952 if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY)
1953 tex->tex.target = TEX_TARGET_2D_ARRAY;
1954 else
1955 return;
1956
1957 Value *x = tex->getSrc(0);
1958 Value *y = tex->getSrc(1);
1959 Value *s = tex->getSrc(arg - 1);
1960
1961 Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA();
1962 Value *ind = tex->getIndirectR();
1963
1964 Value *ms_x = loadMsAdjInfo32(tex->tex.target, 0, slot, ind, tex->tex.bindless);
1965 Value *ms_y = loadMsAdjInfo32(tex->tex.target, 1, slot, ind, tex->tex.bindless);
1966
1967 bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x);
1968 bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y);
1969
1970 s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7));
1971 s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3));
1972
1973 Value *dx = loadMsInfo32(ts, 0x0);
1974 Value *dy = loadMsInfo32(ts, 0x4);
1975
1976 bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx);
1977 bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy);
1978
1979 tex->setSrc(0, tx);
1980 tex->setSrc(1, ty);
1981 tex->moveSources(arg, -1);
1982 }
1983
1984 // Sets 64-bit "generic address", predicate and format sources for SULD/SUST.
1985 // They're computed from the coordinates using the surface info in c[] space.
1986 void
processSurfaceCoordsNVE4(TexInstruction * su)1987 NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su)
1988 {
1989 Instruction *insn;
1990 const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP;
1991 const bool raw =
1992 su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB;
1993 const int slot = su->tex.r;
1994 const int dim = su->tex.target.getDim();
1995 const bool array = su->tex.target.isArray() || su->tex.target.isCube();
1996 const int arg = dim + array;
1997 int c;
1998 Value *zero = bld.mkImm(0);
1999 Value *p1 = NULL;
2000 Value *v;
2001 Value *src[3];
2002 Value *bf, *eau, *off;
2003 Value *addr, *pred;
2004 Value *ind = su->getIndirectR();
2005 Value *y, *z;
2006
2007 off = bld.getScratch(4);
2008 bf = bld.getScratch(4);
2009 addr = bld.getSSA(8);
2010 pred = bld.getScratch(1, FILE_PREDICATE);
2011
2012 bld.setPosition(su, false);
2013
2014 adjustCoordinatesMS(su);
2015
2016 // calculate clamped coordinates
2017 for (c = 0; c < arg; ++c) {
2018 int dimc = c;
2019
2020 if (c == 1 && su->tex.target == TEX_TARGET_1D_ARRAY) {
2021 // The array index is stored in the Z component for 1D arrays.
2022 dimc = 2;
2023 }
2024
2025 src[c] = bld.getScratch();
2026 if (c == 0 && raw)
2027 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_RAW_X, su->tex.bindless);
2028 else
2029 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(dimc), su->tex.bindless);
2030 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero)
2031 ->subOp = getSuClampSubOp(su, dimc);
2032 }
2033 for (; c < 3; ++c)
2034 src[c] = zero;
2035
2036 if (dim == 2 && !array) {
2037 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless);
2038 src[2] = bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(),
2039 v, bld.loadImm(NULL, 16));
2040
2041 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(2), su->tex.bindless);
2042 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[2], src[2], v, zero)
2043 ->subOp = NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
2044 }
2045
2046 // set predicate output
2047 if (su->tex.target == TEX_TARGET_BUFFER) {
2048 src[0]->getInsn()->setFlagsDef(1, pred);
2049 } else
2050 if (array) {
2051 p1 = bld.getSSA(1, FILE_PREDICATE);
2052 src[dim]->getInsn()->setFlagsDef(1, p1);
2053 }
2054
2055 // calculate pixel offset
2056 if (dim == 1) {
2057 y = z = zero;
2058 if (su->tex.target != TEX_TARGET_BUFFER)
2059 bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff));
2060 } else {
2061 y = src[1];
2062 z = src[2];
2063
2064 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless);
2065 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1])
2066 ->subOp = NV50_IR_SUBOP_MADSP(4,4,8); // u16l u16l u16l
2067
2068 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_PITCH, su->tex.bindless);
2069 bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0])
2070 ->subOp = array ?
2071 NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l
2072 }
2073
2074 // calculate effective address part 1
2075 if (su->tex.target == TEX_TARGET_BUFFER) {
2076 if (raw) {
2077 bf = src[0];
2078 } else {
2079 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT, su->tex.bindless);
2080 bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero)
2081 ->subOp = NV50_IR_SUBOP_V1(7,6,8|2);
2082 }
2083 } else {
2084 uint16_t subOp = 0;
2085
2086 switch (dim) {
2087 case 1:
2088 break;
2089 case 2:
2090 if (array) {
2091 z = off;
2092 } else {
2093 subOp = NV50_IR_SUBOP_SUBFM_3D;
2094 }
2095 break;
2096 default:
2097 subOp = NV50_IR_SUBOP_SUBFM_3D;
2098 assert(dim == 3);
2099 break;
2100 }
2101 insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z);
2102 insn->subOp = subOp;
2103 insn->setFlagsDef(1, pred);
2104 }
2105
2106 // part 2
2107 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless);
2108
2109 if (su->tex.target == TEX_TARGET_BUFFER) {
2110 eau = v;
2111 } else {
2112 eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v);
2113 }
2114 // add array layer offset
2115 if (array) {
2116 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY, su->tex.bindless);
2117 if (dim == 1)
2118 bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau)
2119 ->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32
2120 else
2121 bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau)
2122 ->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32
2123 // combine predicates
2124 assert(p1);
2125 bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1);
2126 }
2127
2128 if (atom) {
2129 Value *lo = bf;
2130 if (su->tex.target == TEX_TARGET_BUFFER) {
2131 lo = zero;
2132 bld.mkMov(off, bf);
2133 }
2134 // bf == g[] address & 0xff
2135 // eau == g[] address >> 8
2136 bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau);
2137 bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau);
2138 } else
2139 if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) {
2140 // Convert from u32 to u8 address format, which is what the library code
2141 // doing SULDP currently uses.
2142 // XXX: can SUEAU do this ?
2143 // XXX: does it matter that we don't mask high bytes in bf ?
2144 // Grrr.
2145 bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8));
2146 bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off);
2147 }
2148
2149 bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau);
2150
2151 if (atom && su->tex.target == TEX_TARGET_BUFFER)
2152 bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off);
2153
2154 // let's just set it 0 for raw access and hope it works
2155 v = raw ?
2156 bld.mkImm(0) : loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT, su->tex.bindless);
2157
2158 // get rid of old coordinate sources, make space for fmt info and predicate
2159 su->moveSources(arg, 3 - arg);
2160 // set 64 bit address and 32-bit format sources
2161 su->setSrc(0, addr);
2162 su->setSrc(1, v);
2163 su->setSrc(2, pred);
2164 su->setIndirectR(NULL);
2165
2166 // prevent read fault when the image is not actually bound
2167 CmpInstruction *pred1 =
2168 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2169 TYPE_U32, bld.mkImm(0),
2170 loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless));
2171
2172 if (su->op != OP_SUSTP && su->tex.format) {
2173 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2174 int blockwidth = format->bits[0] + format->bits[1] +
2175 format->bits[2] + format->bits[3];
2176
2177 // make sure that the format doesn't mismatch
2178 assert(format->components != 0);
2179 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred1->getDef(0),
2180 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
2181 loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless),
2182 pred1->getDef(0));
2183 }
2184 su->setPredicate(CC_NOT_P, pred1->getDef(0));
2185
2186 // TODO: initialize def values to 0 when the surface operation is not
2187 // performed (not needed for stores). Also, fix the "address bounds test"
2188 // subtests from arb_shader_image_load_store-invalid for buffers, because it
2189 // seems like that the predicate is not correctly set by suclamp.
2190 }
2191
2192 static DataType
getSrcType(const TexInstruction::ImgFormatDesc * t,int c)2193 getSrcType(const TexInstruction::ImgFormatDesc *t, int c)
2194 {
2195 switch (t->type) {
2196 case FLOAT: return t->bits[c] == 16 ? TYPE_F16 : TYPE_F32;
2197 case UNORM: return t->bits[c] == 8 ? TYPE_U8 : TYPE_U16;
2198 case SNORM: return t->bits[c] == 8 ? TYPE_S8 : TYPE_S16;
2199 case UINT:
2200 return (t->bits[c] == 8 ? TYPE_U8 :
2201 (t->bits[c] == 16 ? TYPE_U16 : TYPE_U32));
2202 case SINT:
2203 return (t->bits[c] == 8 ? TYPE_S8 :
2204 (t->bits[c] == 16 ? TYPE_S16 : TYPE_S32));
2205 }
2206 return TYPE_NONE;
2207 }
2208
2209 static DataType
getDestType(const ImgType type)2210 getDestType(const ImgType type) {
2211 switch (type) {
2212 case FLOAT:
2213 case UNORM:
2214 case SNORM:
2215 return TYPE_F32;
2216 case UINT:
2217 return TYPE_U32;
2218 case SINT:
2219 return TYPE_S32;
2220 default:
2221 assert(!"Impossible type");
2222 return TYPE_NONE;
2223 }
2224 }
2225
2226 void
convertSurfaceFormat(TexInstruction * su,Instruction ** loaded)2227 NVC0LoweringPass::convertSurfaceFormat(TexInstruction *su, Instruction **loaded)
2228 {
2229 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2230 int width = format->bits[0] + format->bits[1] +
2231 format->bits[2] + format->bits[3];
2232 Value *untypedDst[4] = {};
2233 Value *typedDst[4] = {};
2234
2235 // We must convert this to a generic load.
2236 su->op = OP_SULDB;
2237
2238 su->dType = typeOfSize(width / 8);
2239 su->sType = TYPE_U8;
2240
2241 for (int i = 0; i < width / 32; i++)
2242 untypedDst[i] = bld.getSSA();
2243 if (width < 32)
2244 untypedDst[0] = bld.getSSA();
2245
2246 if (loaded && loaded[0]) {
2247 for (int i = 0; i < 4; i++) {
2248 if (loaded[i])
2249 typedDst[i] = loaded[i]->getDef(0);
2250 }
2251 } else {
2252 for (int i = 0; i < 4; i++) {
2253 typedDst[i] = su->getDef(i);
2254 }
2255 }
2256
2257 // Set the untyped dsts as the su's destinations
2258 if (loaded && loaded[0]) {
2259 for (int i = 0; i < 4; i++)
2260 if (loaded[i])
2261 loaded[i]->setDef(0, untypedDst[i]);
2262 } else {
2263 for (int i = 0; i < 4; i++)
2264 su->setDef(i, untypedDst[i]);
2265
2266 bld.setPosition(su, true);
2267 }
2268
2269 // Unpack each component into the typed dsts
2270 int bits = 0;
2271 for (int i = 0; i < 4; bits += format->bits[i], i++) {
2272 if (!typedDst[i])
2273 continue;
2274
2275 if (loaded && loaded[0])
2276 bld.setPosition(loaded[i], true);
2277
2278 if (i >= format->components) {
2279 if (format->type == FLOAT ||
2280 format->type == UNORM ||
2281 format->type == SNORM)
2282 bld.loadImm(typedDst[i], i == 3 ? 1.0f : 0.0f);
2283 else
2284 bld.loadImm(typedDst[i], i == 3 ? 1 : 0);
2285 continue;
2286 }
2287
2288 // Get just that component's data into the relevant place
2289 if (format->bits[i] == 32)
2290 bld.mkMov(typedDst[i], untypedDst[i]);
2291 else if (format->bits[i] == 16)
2292 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
2293 getSrcType(format, i), untypedDst[i / 2])
2294 ->subOp = (i & 1) << (format->type == FLOAT ? 0 : 1);
2295 else if (format->bits[i] == 8)
2296 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
2297 getSrcType(format, i), untypedDst[0])->subOp = i;
2298 else {
2299 bld.mkOp2(OP_EXTBF, TYPE_U32, typedDst[i], untypedDst[bits / 32],
2300 bld.mkImm((bits % 32) | (format->bits[i] << 8)));
2301 if (format->type == UNORM || format->type == SNORM)
2302 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], getSrcType(format, i), typedDst[i]);
2303 }
2304
2305 // Normalize / convert as necessary
2306 if (format->type == UNORM)
2307 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << format->bits[i]) - 1)));
2308 else if (format->type == SNORM)
2309 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << (format->bits[i] - 1)) - 1)));
2310 else if (format->type == FLOAT && format->bits[i] < 16) {
2311 bld.mkOp2(OP_SHL, TYPE_U32, typedDst[i], typedDst[i], bld.loadImm(NULL, 15 - format->bits[i]));
2312 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], TYPE_F16, typedDst[i]);
2313 }
2314 }
2315
2316 if (format->bgra) {
2317 std::swap(typedDst[0], typedDst[2]);
2318 }
2319 }
2320
2321 void
insertOOBSurfaceOpResult(TexInstruction * su)2322 NVC0LoweringPass::insertOOBSurfaceOpResult(TexInstruction *su)
2323 {
2324 if (!su->getPredicate())
2325 return;
2326
2327 bld.setPosition(su, true);
2328
2329 for (unsigned i = 0; su->defExists(i); ++i) {
2330 ValueDef &def = su->def(i);
2331
2332 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2333 assert(su->cc == CC_NOT_P);
2334 mov->setPredicate(CC_P, su->getPredicate());
2335 Instruction *uni = bld.mkOp2(OP_UNION, TYPE_U32, bld.getSSA(), NULL, mov->getDef(0));
2336
2337 def.replace(uni->getDef(0), false);
2338 uni->setSrc(0, def.get());
2339 }
2340 }
2341
2342 void
handleSurfaceOpNVE4(TexInstruction * su)2343 NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su)
2344 {
2345 processSurfaceCoordsNVE4(su);
2346
2347 if (su->op == OP_SULDP) {
2348 convertSurfaceFormat(su, NULL);
2349 insertOOBSurfaceOpResult(su);
2350 }
2351
2352 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
2353 assert(su->getPredicate());
2354 Value *pred =
2355 bld.mkOp2v(OP_OR, TYPE_U8, bld.getScratch(1, FILE_PREDICATE),
2356 su->getPredicate(), su->getSrc(2));
2357
2358 Instruction *red = bld.mkOp(OP_ATOM, su->dType, bld.getSSA());
2359 red->subOp = su->subOp;
2360 red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0));
2361 red->setSrc(1, su->getSrc(3));
2362 if (su->subOp == NV50_IR_SUBOP_ATOM_CAS)
2363 red->setSrc(2, su->getSrc(4));
2364 red->setIndirect(0, 0, su->getSrc(0));
2365
2366 // make sure to initialize dst value when the atomic operation is not
2367 // performed
2368 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2369
2370 assert(su->cc == CC_NOT_P);
2371 red->setPredicate(su->cc, pred);
2372 mov->setPredicate(CC_P, pred);
2373
2374 bld.mkOp2(OP_UNION, TYPE_U32, su->getDef(0),
2375 red->getDef(0), mov->getDef(0));
2376
2377 delete_Instruction(bld.getProgram(), su);
2378 handleCasExch(red, true);
2379 }
2380
2381 if (su->op == OP_SUSTB || su->op == OP_SUSTP)
2382 su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8;
2383 }
2384
2385 void
processSurfaceCoordsNVC0(TexInstruction * su)2386 NVC0LoweringPass::processSurfaceCoordsNVC0(TexInstruction *su)
2387 {
2388 const int slot = su->tex.r;
2389 const int dim = su->tex.target.getDim();
2390 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2391 int c;
2392 Value *zero = bld.mkImm(0);
2393 Value *src[3];
2394 Value *v;
2395 Value *ind = su->getIndirectR();
2396
2397 bld.setPosition(su, false);
2398
2399 adjustCoordinatesMS(su);
2400
2401 if (ind) {
2402 Value *ptr;
2403 ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ind, bld.mkImm(su->tex.r));
2404 ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7));
2405 su->setIndirectR(ptr);
2406 }
2407
2408 // get surface coordinates
2409 for (c = 0; c < arg; ++c)
2410 src[c] = su->getSrc(c);
2411 for (; c < 3; ++c)
2412 src[c] = zero;
2413
2414 // calculate pixel offset
2415 if (su->op == OP_SULDP || su->op == OP_SUREDP) {
2416 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless);
2417 su->setSrc(0, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[0], v));
2418 }
2419
2420 // add array layer offset
2421 if (su->tex.target.isArray() || su->tex.target.isCube()) {
2422 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY, su->tex.bindless);
2423 assert(dim > 1);
2424 su->setSrc(2, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[2], v));
2425 }
2426
2427 // prevent read fault when the image is not actually bound
2428 CmpInstruction *pred =
2429 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2430 TYPE_U32, bld.mkImm(0),
2431 loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless));
2432 if (su->op != OP_SUSTP && su->tex.format) {
2433 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2434 int blockwidth = format->bits[0] + format->bits[1] +
2435 format->bits[2] + format->bits[3];
2436
2437 assert(format->components != 0);
2438 // make sure that the format doesn't mismatch when it's not FMT_NONE
2439 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0),
2440 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
2441 loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless),
2442 pred->getDef(0));
2443 }
2444 su->setPredicate(CC_NOT_P, pred->getDef(0));
2445 }
2446
2447 void
handleSurfaceOpNVC0(TexInstruction * su)2448 NVC0LoweringPass::handleSurfaceOpNVC0(TexInstruction *su)
2449 {
2450 if (su->tex.target == TEX_TARGET_1D_ARRAY) {
2451 /* As 1d arrays also need 3 coordinates, switching to TEX_TARGET_2D_ARRAY
2452 * will simplify the lowering pass and the texture constraints. */
2453 su->moveSources(1, 1);
2454 su->setSrc(1, bld.loadImm(NULL, 0));
2455 su->tex.target = TEX_TARGET_2D_ARRAY;
2456 }
2457
2458 processSurfaceCoordsNVC0(su);
2459
2460 if (su->op == OP_SULDP) {
2461 convertSurfaceFormat(su, NULL);
2462 insertOOBSurfaceOpResult(su);
2463 }
2464
2465 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
2466 const int dim = su->tex.target.getDim();
2467 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2468 LValue *addr = bld.getSSA(8);
2469 Value *def = su->getDef(0);
2470
2471 su->op = OP_SULEA;
2472
2473 // Set the destination to the address
2474 su->dType = TYPE_U64;
2475 su->setDef(0, addr);
2476 su->setDef(1, su->getPredicate());
2477
2478 bld.setPosition(su, true);
2479
2480 // Perform the atomic op
2481 Instruction *red = bld.mkOp(OP_ATOM, su->sType, bld.getSSA());
2482 red->subOp = su->subOp;
2483 red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, su->sType, 0));
2484 red->setSrc(1, su->getSrc(arg));
2485 if (red->subOp == NV50_IR_SUBOP_ATOM_CAS)
2486 red->setSrc(2, su->getSrc(arg + 1));
2487 red->setIndirect(0, 0, addr);
2488
2489 // make sure to initialize dst value when the atomic operation is not
2490 // performed
2491 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2492
2493 assert(su->cc == CC_NOT_P);
2494 red->setPredicate(su->cc, su->getPredicate());
2495 mov->setPredicate(CC_P, su->getPredicate());
2496
2497 bld.mkOp2(OP_UNION, TYPE_U32, def, red->getDef(0), mov->getDef(0));
2498
2499 handleCasExch(red, false);
2500 }
2501 }
2502
2503 TexInstruction *
processSurfaceCoordsGM107(TexInstruction * su,Instruction * ret[4])2504 NVC0LoweringPass::processSurfaceCoordsGM107(TexInstruction *su, Instruction *ret[4])
2505 {
2506 const int slot = su->tex.r;
2507 const int dim = su->tex.target.getDim();
2508 const bool array = su->tex.target.isArray() || su->tex.target.isCube();
2509 const int arg = dim + array;
2510 Value *ind = su->getIndirectR();
2511 Value *handle;
2512 Instruction *pred = NULL, *pred2d = NULL;
2513 int pos = 0;
2514
2515 bld.setPosition(su, false);
2516
2517 adjustCoordinatesMS(su);
2518
2519 // add texture handle
2520 switch (su->op) {
2521 case OP_SUSTP:
2522 pos = 4;
2523 break;
2524 case OP_SUREDP:
2525 pos = (su->subOp == NV50_IR_SUBOP_ATOM_CAS) ? 2 : 1;
2526 break;
2527 default:
2528 assert(pos == 0);
2529 break;
2530 }
2531
2532 if (dim == 2 && !array) {
2533 // This might be a 2d slice of a 3d texture, try to load the z
2534 // coordinate in.
2535 Value *v;
2536 if (!su->tex.bindless)
2537 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless);
2538 else
2539 v = bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), ind, bld.mkImm(11));
2540 Value *is_3d = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), v, bld.mkImm(1));
2541 pred2d = bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2542 TYPE_U32, bld.mkImm(0), is_3d);
2543
2544 bld.mkOp2(OP_SHR, TYPE_U32, v, v, bld.loadImm(NULL, 16));
2545 su->moveSources(dim, 1);
2546 su->setSrc(dim, v);
2547 su->tex.target = nv50_ir::TEX_TARGET_3D;
2548 pos++;
2549 }
2550
2551 if (su->tex.bindless)
2552 handle = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ind, bld.mkImm(2047));
2553 else
2554 handle = loadTexHandle(ind, slot + 32);
2555
2556 su->setSrc(arg + pos, handle);
2557
2558 // The address check doesn't make sense here. The format check could make
2559 // sense but it's a bit of a pain.
2560 if (!su->tex.bindless) {
2561 // prevent read fault when the image is not actually bound
2562 pred =
2563 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2564 TYPE_U32, bld.mkImm(0),
2565 loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless));
2566 if (su->op != OP_SUSTP && su->tex.format) {
2567 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2568 int blockwidth = format->bits[0] + format->bits[1] +
2569 format->bits[2] + format->bits[3];
2570
2571 assert(format->components != 0);
2572 // make sure that the format doesn't mismatch when it's not FMT_NONE
2573 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0),
2574 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
2575 loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless),
2576 pred->getDef(0));
2577 }
2578 }
2579
2580 // Now we have "pred" which (optionally) contains whether to do the surface
2581 // op at all, and a "pred2d" which indicates that, in case of doing the
2582 // surface op, we have to create a 2d and 3d version, conditioned on pred2d.
2583 TexInstruction *su2d = NULL;
2584 if (pred2d) {
2585 su2d = cloneForward(func, su)->asTex();
2586 for (unsigned i = 0; su->defExists(i); ++i)
2587 su2d->setDef(i, bld.getSSA());
2588 su2d->moveSources(dim + 1, -1);
2589 su2d->tex.target = nv50_ir::TEX_TARGET_2D;
2590 }
2591 if (pred2d && pred) {
2592 Instruction *pred3d = bld.mkOp2(OP_AND, TYPE_U8,
2593 bld.getSSA(1, FILE_PREDICATE),
2594 pred->getDef(0), pred2d->getDef(0));
2595 pred3d->src(0).mod = Modifier(NV50_IR_MOD_NOT);
2596 pred3d->src(1).mod = Modifier(NV50_IR_MOD_NOT);
2597 su->setPredicate(CC_P, pred3d->getDef(0));
2598 pred2d = bld.mkOp2(OP_AND, TYPE_U8, bld.getSSA(1, FILE_PREDICATE),
2599 pred->getDef(0), pred2d->getDef(0));
2600 pred2d->src(0).mod = Modifier(NV50_IR_MOD_NOT);
2601 } else if (pred) {
2602 su->setPredicate(CC_NOT_P, pred->getDef(0));
2603 } else if (pred2d) {
2604 su->setPredicate(CC_NOT_P, pred2d->getDef(0));
2605 }
2606 if (su2d) {
2607 su2d->setPredicate(CC_P, pred2d->getDef(0));
2608 bld.insert(su2d);
2609
2610 // Create a UNION so that RA assigns the same registers
2611 bld.setPosition(su, true);
2612 for (unsigned i = 0; su->defExists(i); ++i) {
2613 assert(i < 4);
2614
2615 ValueDef &def = su->def(i);
2616 ValueDef &def2 = su2d->def(i);
2617 Instruction *mov = NULL;
2618
2619 if (pred) {
2620 mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2621 mov->setPredicate(CC_P, pred->getDef(0));
2622 }
2623
2624 Instruction *uni = ret[i] = bld.mkOp2(OP_UNION, TYPE_U32,
2625 bld.getSSA(),
2626 NULL, def2.get());
2627 def.replace(uni->getDef(0), false);
2628 uni->setSrc(0, def.get());
2629 if (mov)
2630 uni->setSrc(2, mov->getDef(0));
2631 }
2632 } else if (pred) {
2633 // Create a UNION so that RA assigns the same registers
2634 bld.setPosition(su, true);
2635 for (unsigned i = 0; su->defExists(i); ++i) {
2636 assert(i < 4);
2637
2638 ValueDef &def = su->def(i);
2639
2640 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2641 mov->setPredicate(CC_P, pred->getDef(0));
2642
2643 Instruction *uni = ret[i] = bld.mkOp2(OP_UNION, TYPE_U32,
2644 bld.getSSA(),
2645 NULL, mov->getDef(0));
2646 def.replace(uni->getDef(0), false);
2647 uni->setSrc(0, def.get());
2648 }
2649 }
2650
2651 return su2d;
2652 }
2653
2654 void
handleSurfaceOpGM107(TexInstruction * su)2655 NVC0LoweringPass::handleSurfaceOpGM107(TexInstruction *su)
2656 {
2657 // processSurfaceCoords also takes care of fixing up the outputs and
2658 // union'ing them with 0 as necessary. Additionally it may create a second
2659 // surface which needs some of the similar fixups.
2660
2661 Instruction *loaded[4] = {};
2662 TexInstruction *su2 = processSurfaceCoordsGM107(su, loaded);
2663
2664 if (su->op == OP_SULDP) {
2665 convertSurfaceFormat(su, loaded);
2666 }
2667
2668 if (su->op == OP_SUREDP) {
2669 su->op = OP_SUREDB;
2670 }
2671
2672 // If we fixed up the type of the regular surface load instruction, we also
2673 // have to fix up the copy.
2674 if (su2) {
2675 su2->op = su->op;
2676 su2->dType = su->dType;
2677 su2->sType = su->sType;
2678 }
2679 }
2680
2681 bool
handleWRSV(Instruction * i)2682 NVC0LoweringPass::handleWRSV(Instruction *i)
2683 {
2684 Instruction *st;
2685 Symbol *sym;
2686 uint32_t addr;
2687
2688 // must replace, $sreg are not writeable
2689 addr = targ->getSVAddress(FILE_SHADER_OUTPUT, i->getSrc(0)->asSym());
2690 if (addr >= 0x400)
2691 return false;
2692 sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
2693
2694 st = bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0),
2695 i->getSrc(1));
2696 st->perPatch = i->perPatch;
2697
2698 bld.getBB()->remove(i);
2699 return true;
2700 }
2701
2702 void
handleLDST(Instruction * i)2703 NVC0LoweringPass::handleLDST(Instruction *i)
2704 {
2705 if (i->src(0).getFile() == FILE_SHADER_INPUT) {
2706 if (prog->getType() == Program::TYPE_COMPUTE) {
2707 i->getSrc(0)->reg.file = FILE_MEMORY_CONST;
2708 i->getSrc(0)->reg.fileIndex = 0;
2709 } else
2710 if (prog->getType() == Program::TYPE_GEOMETRY &&
2711 i->src(0).isIndirect(0)) {
2712 // XXX: this assumes vec4 units
2713 Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2714 i->getIndirect(0, 0), bld.mkImm(4));
2715 i->setIndirect(0, 0, ptr);
2716 i->op = OP_VFETCH;
2717 } else {
2718 i->op = OP_VFETCH;
2719 assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP
2720 }
2721 } else if (i->src(0).getFile() == FILE_MEMORY_CONST) {
2722 int8_t fileIndex = i->getSrc(0)->reg.fileIndex - 1;
2723 Value *ind = i->getIndirect(0, 1);
2724
2725 if (targ->getChipset() >= NVISA_GK104_CHIPSET &&
2726 prog->getType() == Program::TYPE_COMPUTE &&
2727 (fileIndex >= 6 || ind)) {
2728 // The launch descriptor only allows to set up 8 CBs, but OpenGL
2729 // requires at least 12 UBOs. To bypass this limitation, for constant
2730 // buffers 7+, we store the addrs into the driver constbuf and we
2731 // directly load from the global memory.
2732 if (ind) {
2733 // Clamp the UBO index when an indirect access is used to avoid
2734 // loading information from the wrong place in the driver cb.
2735 // TODO - synchronize the max with the driver.
2736 ind = bld.mkOp2v(OP_MIN, TYPE_U32, bld.getSSA(),
2737 bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(),
2738 ind, bld.loadImm(NULL, fileIndex)),
2739 bld.loadImm(NULL, 13));
2740 fileIndex = 0;
2741 }
2742
2743 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2744 Value *ptr = loadUboInfo64(ind, fileIndex * 16);
2745 Value *length = loadUboLength32(ind, fileIndex * 16);
2746 Value *pred = new_LValue(func, FILE_PREDICATE);
2747 if (i->src(0).isIndirect(0)) {
2748 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2749 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2750 }
2751 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2752 i->setIndirect(0, 1, NULL);
2753 i->setIndirect(0, 0, ptr);
2754 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2755 i->setPredicate(CC_NOT_P, pred);
2756 Value *zero, *dst = i->getDef(0);
2757 i->setDef(0, bld.getSSA());
2758
2759 bld.setPosition(i, true);
2760 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
2761 ->setPredicate(CC_P, pred);
2762 bld.mkOp2(OP_UNION, TYPE_U32, dst, i->getDef(0), zero);
2763 } else if (i->src(0).isIndirect(1)) {
2764 Value *ptr;
2765 if (i->src(0).isIndirect(0))
2766 ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(),
2767 i->getIndirect(0, 1), bld.mkImm(0x1010),
2768 i->getIndirect(0, 0));
2769 else
2770 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2771 i->getIndirect(0, 1), bld.mkImm(16));
2772 i->setIndirect(0, 1, NULL);
2773 i->setIndirect(0, 0, ptr);
2774 i->subOp = NV50_IR_SUBOP_LDC_IS;
2775 }
2776 } else if (i->src(0).getFile() == FILE_SHADER_OUTPUT) {
2777 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2778 i->op = OP_VFETCH;
2779 } else if (i->src(0).getFile() == FILE_MEMORY_BUFFER) {
2780 Value *ind = i->getIndirect(0, 1);
2781 Value *ptr = loadBufInfo64(ind, i->getSrc(0)->reg.fileIndex * 16);
2782 // XXX come up with a way not to do this for EVERY little access but
2783 // rather to batch these up somehow. Unfortunately we've lost the
2784 // information about the field width by the time we get here.
2785 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2786 Value *length = loadBufLength32(ind, i->getSrc(0)->reg.fileIndex * 16);
2787 Value *pred = new_LValue(func, FILE_PREDICATE);
2788 if (i->src(0).isIndirect(0)) {
2789 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2790 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2791 }
2792 i->setIndirect(0, 1, NULL);
2793 i->setIndirect(0, 0, ptr);
2794 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2795 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2796 i->setPredicate(CC_NOT_P, pred);
2797 if (i->defExists(0)) {
2798 Value *zero, *dst = i->getDef(0);
2799 i->setDef(0, bld.getSSA());
2800
2801 bld.setPosition(i, true);
2802 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
2803 ->setPredicate(CC_P, pred);
2804 bld.mkOp2(OP_UNION, TYPE_U32, dst, i->getDef(0), zero);
2805 }
2806 }
2807 }
2808
2809 void
readTessCoord(LValue * dst,int c)2810 NVC0LoweringPass::readTessCoord(LValue *dst, int c)
2811 {
2812 Value *laneid = bld.getSSA();
2813 Value *x, *y;
2814
2815 bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0));
2816
2817 if (c == 0) {
2818 x = dst;
2819 y = NULL;
2820 } else
2821 if (c == 1) {
2822 x = NULL;
2823 y = dst;
2824 } else {
2825 assert(c == 2);
2826 if (prog->driver_out->prop.tp.domain != PIPE_PRIM_TRIANGLES) {
2827 bld.mkMov(dst, bld.loadImm(NULL, 0));
2828 return;
2829 }
2830 x = bld.getSSA();
2831 y = bld.getSSA();
2832 }
2833 if (x)
2834 bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid);
2835 if (y)
2836 bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid);
2837
2838 if (c == 2) {
2839 bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y);
2840 bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst);
2841 }
2842 }
2843
2844 bool
handleRDSV(Instruction * i)2845 NVC0LoweringPass::handleRDSV(Instruction *i)
2846 {
2847 Symbol *sym = i->getSrc(0)->asSym();
2848 const SVSemantic sv = sym->reg.data.sv.sv;
2849 Value *vtx = NULL;
2850 Instruction *ld;
2851 uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
2852
2853 if (addr >= 0x400) {
2854 // mov $sreg
2855 if (sym->reg.data.sv.index == 3) {
2856 // TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID
2857 i->op = OP_MOV;
2858 i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0));
2859 } else
2860 if (sv == SV_TID) {
2861 // Help CSE combine TID fetches
2862 Value *tid = bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(),
2863 bld.mkSysVal(SV_COMBINED_TID, 0));
2864 i->op = OP_EXTBF;
2865 i->setSrc(0, tid);
2866 switch (sym->reg.data.sv.index) {
2867 case 0: i->setSrc(1, bld.mkImm(0x1000)); break;
2868 case 1: i->setSrc(1, bld.mkImm(0x0a10)); break;
2869 case 2: i->setSrc(1, bld.mkImm(0x061a)); break;
2870 }
2871 }
2872 if (sv == SV_VERTEX_COUNT) {
2873 bld.setPosition(i, true);
2874 bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0), bld.mkImm(0x808));
2875 }
2876 return true;
2877 }
2878
2879 switch (sv) {
2880 case SV_POSITION:
2881 assert(prog->getType() == Program::TYPE_FRAGMENT);
2882 if (i->srcExists(1)) {
2883 // Pass offset through to the interpolation logic
2884 ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET,
2885 i->getDef(0), addr, NULL);
2886 ld->setSrc(1, i->getSrc(1));
2887 } else {
2888 bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
2889 }
2890 break;
2891 case SV_FACE:
2892 {
2893 Value *face = i->getDef(0);
2894 bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL);
2895 if (i->dType == TYPE_F32) {
2896 bld.mkOp2(OP_OR, TYPE_U32, face, face, bld.mkImm(0x00000001));
2897 bld.mkOp1(OP_NEG, TYPE_S32, face, face);
2898 bld.mkCvt(OP_CVT, TYPE_F32, face, TYPE_S32, face);
2899 }
2900 }
2901 break;
2902 case SV_TESS_COORD:
2903 assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL);
2904 readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index);
2905 break;
2906 case SV_NTID:
2907 case SV_NCTAID:
2908 case SV_GRIDID:
2909 assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise
2910 if (sym->reg.data.sv.index == 3) {
2911 i->op = OP_MOV;
2912 i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1));
2913 return true;
2914 }
2915 // Fallthrough
2916 case SV_WORK_DIM:
2917 addr += prog->driver->prop.cp.gridInfoBase;
2918 bld.mkLoad(TYPE_U32, i->getDef(0),
2919 bld.mkSymbol(FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2920 TYPE_U32, addr), NULL);
2921 break;
2922 case SV_SAMPLE_INDEX:
2923 // TODO: Properly pass source as an address in the PIX address space
2924 // (which can be of the form [r0+offset]). But this is currently
2925 // unnecessary.
2926 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2927 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2928 break;
2929 case SV_SAMPLE_POS: {
2930 Value *sampleID = bld.getScratch();
2931 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, sampleID, bld.mkImm(0));
2932 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2933 Value *offset = calculateSampleOffset(sampleID);
2934
2935 assert(prog->driver_out->prop.fp.readsSampleLocations);
2936
2937 if (targ->getChipset() >= NVISA_GM200_CHIPSET) {
2938 bld.mkLoad(TYPE_F32,
2939 i->getDef(0),
2940 bld.mkSymbol(
2941 FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2942 TYPE_U32, prog->driver->io.sampleInfoBase),
2943 offset);
2944 bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0),
2945 bld.mkImm(0x040c + sym->reg.data.sv.index * 16));
2946 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(0), TYPE_U32, i->getDef(0));
2947 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(0), i->getDef(0), bld.mkImm(1.0f / 16.0f));
2948 } else {
2949 bld.mkLoad(TYPE_F32,
2950 i->getDef(0),
2951 bld.mkSymbol(
2952 FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2953 TYPE_U32, prog->driver->io.sampleInfoBase +
2954 4 * sym->reg.data.sv.index),
2955 offset);
2956 }
2957 break;
2958 }
2959 case SV_SAMPLE_MASK: {
2960 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2961 ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK;
2962 Instruction *sampleid =
2963 bld.mkOp1(OP_PIXLD, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2964 sampleid->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2965 Value *masked =
2966 bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ld->getDef(0),
2967 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2968 bld.loadImm(NULL, 1), sampleid->getDef(0)));
2969 if (prog->persampleInvocation) {
2970 bld.mkMov(i->getDef(0), masked);
2971 } else {
2972 bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), ld->getDef(0), masked,
2973 bld.mkImm(0))
2974 ->subOp = 1;
2975 }
2976 break;
2977 }
2978 case SV_BASEVERTEX:
2979 case SV_BASEINSTANCE:
2980 case SV_DRAWID:
2981 ld = bld.mkLoad(TYPE_U32, i->getDef(0),
2982 bld.mkSymbol(FILE_MEMORY_CONST,
2983 prog->driver->io.auxCBSlot,
2984 TYPE_U32,
2985 prog->driver->io.drawInfoBase +
2986 4 * (sv - SV_BASEVERTEX)),
2987 NULL);
2988 break;
2989 default:
2990 if (prog->getType() == Program::TYPE_TESSELLATION_EVAL && !i->perPatch)
2991 vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2992 if (prog->getType() == Program::TYPE_FRAGMENT) {
2993 bld.mkInterp(NV50_IR_INTERP_FLAT, i->getDef(0), addr, NULL);
2994 } else {
2995 ld = bld.mkFetch(i->getDef(0), i->dType,
2996 FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx);
2997 ld->perPatch = i->perPatch;
2998 }
2999 break;
3000 }
3001 bld.getBB()->remove(i);
3002 return true;
3003 }
3004
3005 bool
handleDIV(Instruction * i)3006 NVC0LoweringPass::handleDIV(Instruction *i)
3007 {
3008 if (!isFloatType(i->dType))
3009 return true;
3010 bld.setPosition(i, false);
3011 Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(typeSizeof(i->dType)), i->getSrc(1));
3012 i->op = OP_MUL;
3013 i->setSrc(1, rcp->getDef(0));
3014 return true;
3015 }
3016
3017 bool
handleMOD(Instruction * i)3018 NVC0LoweringPass::handleMOD(Instruction *i)
3019 {
3020 if (!isFloatType(i->dType))
3021 return true;
3022 LValue *value = bld.getScratch(typeSizeof(i->dType));
3023 bld.mkOp1(OP_RCP, i->dType, value, i->getSrc(1));
3024 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(0), value);
3025 bld.mkOp1(OP_TRUNC, i->dType, value, value);
3026 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(1), value);
3027 i->op = OP_SUB;
3028 i->setSrc(1, value);
3029 return true;
3030 }
3031
3032 bool
handleSQRT(Instruction * i)3033 NVC0LoweringPass::handleSQRT(Instruction *i)
3034 {
3035 if (targ->isOpSupported(OP_SQRT, i->dType))
3036 return true;
3037
3038 if (i->dType == TYPE_F64) {
3039 Value *pred = bld.getSSA(1, FILE_PREDICATE);
3040 Value *zero = bld.loadImm(NULL, 0.0);
3041 Value *dst = bld.getSSA(8);
3042 bld.mkOp1(OP_RSQ, i->dType, dst, i->getSrc(0));
3043 bld.mkCmp(OP_SET, CC_LE, i->dType, pred, i->dType, i->getSrc(0), zero);
3044 bld.mkOp3(OP_SELP, TYPE_U64, dst, zero, dst, pred);
3045 i->op = OP_MUL;
3046 i->setSrc(1, dst);
3047 // TODO: Handle this properly with a library function
3048 } else {
3049 bld.setPosition(i, true);
3050 i->op = OP_RSQ;
3051 bld.mkOp1(OP_RCP, i->dType, i->getDef(0), i->getDef(0));
3052 }
3053
3054 return true;
3055 }
3056
3057 bool
handlePOW(Instruction * i)3058 NVC0LoweringPass::handlePOW(Instruction *i)
3059 {
3060 LValue *val = bld.getScratch();
3061
3062 bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
3063 bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
3064 bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
3065
3066 i->op = OP_EX2;
3067 i->setSrc(0, val);
3068 i->setSrc(1, NULL);
3069
3070 return true;
3071 }
3072
3073 bool
handleEXPORT(Instruction * i)3074 NVC0LoweringPass::handleEXPORT(Instruction *i)
3075 {
3076 if (prog->getType() == Program::TYPE_FRAGMENT) {
3077 int id = i->getSrc(0)->reg.data.offset / 4;
3078
3079 if (i->src(0).isIndirect(0)) // TODO, ugly
3080 return false;
3081 i->op = OP_MOV;
3082 i->subOp = NV50_IR_SUBOP_MOV_FINAL;
3083 i->src(0).set(i->src(1));
3084 i->setSrc(1, NULL);
3085 i->setDef(0, new_LValue(func, FILE_GPR));
3086 i->getDef(0)->reg.data.id = id;
3087
3088 prog->maxGPR = MAX2(prog->maxGPR, id);
3089 } else
3090 if (prog->getType() == Program::TYPE_GEOMETRY) {
3091 i->setIndirect(0, 1, gpEmitAddress);
3092 }
3093 return true;
3094 }
3095
3096 bool
handleOUT(Instruction * i)3097 NVC0LoweringPass::handleOUT(Instruction *i)
3098 {
3099 Instruction *prev = i->prev;
3100 ImmediateValue stream, prevStream;
3101
3102 // Only merge if the stream ids match. Also, note that the previous
3103 // instruction would have already been lowered, so we take arg1 from it.
3104 if (i->op == OP_RESTART && prev && prev->op == OP_EMIT &&
3105 i->src(0).getImmediate(stream) &&
3106 prev->src(1).getImmediate(prevStream) &&
3107 stream.reg.data.u32 == prevStream.reg.data.u32) {
3108 i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART;
3109 delete_Instruction(prog, i);
3110 } else {
3111 assert(gpEmitAddress);
3112 i->setDef(0, gpEmitAddress);
3113 i->setSrc(1, i->getSrc(0));
3114 i->setSrc(0, gpEmitAddress);
3115 }
3116 return true;
3117 }
3118
3119 Value *
calculateSampleOffset(Value * sampleID)3120 NVC0LoweringPass::calculateSampleOffset(Value *sampleID)
3121 {
3122 Value *offset = bld.getScratch();
3123 if (targ->getChipset() >= NVISA_GM200_CHIPSET) {
3124 // Sample location offsets (in bytes) are calculated like so:
3125 // offset = (SV_POSITION.y % 4 * 2) + (SV_POSITION.x % 2)
3126 // offset = offset * 32 + sampleID % 8 * 4;
3127 // which is equivalent to:
3128 // offset = (SV_POSITION.y & 0x3) << 6 + (SV_POSITION.x & 0x1) << 5;
3129 // offset += sampleID << 2
3130
3131 // The second operand (src1) of the INSBF instructions are like so:
3132 // 0xssll where ss is the size and ll is the offset.
3133 // so: dest = src2 | (src0 & (1 << ss - 1)) << ll
3134
3135 // Add sample ID (offset = (sampleID & 0x7) << 2)
3136 bld.mkOp3(OP_INSBF, TYPE_U32, offset, sampleID, bld.mkImm(0x0302), bld.mkImm(0x0));
3137
3138 Symbol *xSym = bld.mkSysVal(SV_POSITION, 0);
3139 Symbol *ySym = bld.mkSysVal(SV_POSITION, 1);
3140 Value *coord = bld.getScratch();
3141
3142 // Add X coordinate (offset |= (SV_POSITION.x & 0x1) << 5)
3143 bld.mkInterp(NV50_IR_INTERP_LINEAR, coord,
3144 targ->getSVAddress(FILE_SHADER_INPUT, xSym), NULL);
3145 bld.mkCvt(OP_CVT, TYPE_U32, coord, TYPE_F32, coord)
3146 ->rnd = ROUND_ZI;
3147 bld.mkOp3(OP_INSBF, TYPE_U32, offset, coord, bld.mkImm(0x0105), offset);
3148
3149 // Add Y coordinate (offset |= (SV_POSITION.y & 0x3) << 6)
3150 bld.mkInterp(NV50_IR_INTERP_LINEAR, coord,
3151 targ->getSVAddress(FILE_SHADER_INPUT, ySym), NULL);
3152 bld.mkCvt(OP_CVT, TYPE_U32, coord, TYPE_F32, coord)
3153 ->rnd = ROUND_ZI;
3154 bld.mkOp3(OP_INSBF, TYPE_U32, offset, coord, bld.mkImm(0x0206), offset);
3155 } else {
3156 bld.mkOp2(OP_SHL, TYPE_U32, offset, sampleID, bld.mkImm(3));
3157 }
3158 return offset;
3159 }
3160
3161 // Handle programmable sample locations for GM20x+
3162 void
handlePIXLD(Instruction * i)3163 NVC0LoweringPass::handlePIXLD(Instruction *i)
3164 {
3165 if (i->subOp != NV50_IR_SUBOP_PIXLD_OFFSET)
3166 return;
3167 if (targ->getChipset() < NVISA_GM200_CHIPSET)
3168 return;
3169
3170 assert(prog->driver_out->prop.fp.readsSampleLocations);
3171
3172 bld.mkLoad(TYPE_F32,
3173 i->getDef(0),
3174 bld.mkSymbol(
3175 FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
3176 TYPE_U32, prog->driver->io.sampleInfoBase),
3177 calculateSampleOffset(i->getSrc(0)));
3178
3179 bld.getBB()->remove(i);
3180 }
3181
3182 // Generate a binary predicate if an instruction is predicated by
3183 // e.g. an f32 value.
3184 void
checkPredicate(Instruction * insn)3185 NVC0LoweringPass::checkPredicate(Instruction *insn)
3186 {
3187 Value *pred = insn->getPredicate();
3188 Value *pdst;
3189
3190 if (!pred || pred->reg.file == FILE_PREDICATE)
3191 return;
3192 pdst = new_LValue(func, FILE_PREDICATE);
3193
3194 // CAUTION: don't use pdst->getInsn, the definition might not be unique,
3195 // delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass
3196
3197 bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred);
3198
3199 insn->setPredicate(insn->cc, pdst);
3200 }
3201
3202 //
3203 // - add quadop dance for texturing
3204 // - put FP outputs in GPRs
3205 // - convert instruction sequences
3206 //
3207 bool
visit(Instruction * i)3208 NVC0LoweringPass::visit(Instruction *i)
3209 {
3210 bool ret = true;
3211 bld.setPosition(i, false);
3212
3213 if (i->cc != CC_ALWAYS)
3214 checkPredicate(i);
3215
3216 switch (i->op) {
3217 case OP_TEX:
3218 case OP_TXB:
3219 case OP_TXL:
3220 case OP_TXF:
3221 case OP_TXG:
3222 return handleTEX(i->asTex());
3223 case OP_TXD:
3224 return handleTXD(i->asTex());
3225 case OP_TXLQ:
3226 return handleTXLQ(i->asTex());
3227 case OP_TXQ:
3228 return handleTXQ(i->asTex());
3229 case OP_EX2:
3230 bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
3231 i->setSrc(0, i->getDef(0));
3232 break;
3233 case OP_POW:
3234 return handlePOW(i);
3235 case OP_DIV:
3236 return handleDIV(i);
3237 case OP_MOD:
3238 return handleMOD(i);
3239 case OP_SQRT:
3240 return handleSQRT(i);
3241 case OP_EXPORT:
3242 ret = handleEXPORT(i);
3243 break;
3244 case OP_EMIT:
3245 case OP_RESTART:
3246 return handleOUT(i);
3247 case OP_RDSV:
3248 return handleRDSV(i);
3249 case OP_WRSV:
3250 return handleWRSV(i);
3251 case OP_STORE:
3252 case OP_LOAD:
3253 handleLDST(i);
3254 break;
3255 case OP_ATOM:
3256 {
3257 const bool cctl = i->src(0).getFile() == FILE_MEMORY_BUFFER;
3258 handleATOM(i);
3259 handleCasExch(i, cctl);
3260 }
3261 break;
3262 case OP_SULDB:
3263 case OP_SULDP:
3264 case OP_SUSTB:
3265 case OP_SUSTP:
3266 case OP_SUREDB:
3267 case OP_SUREDP:
3268 if (targ->getChipset() >= NVISA_GM107_CHIPSET)
3269 handleSurfaceOpGM107(i->asTex());
3270 else if (targ->getChipset() >= NVISA_GK104_CHIPSET)
3271 handleSurfaceOpNVE4(i->asTex());
3272 else
3273 handleSurfaceOpNVC0(i->asTex());
3274 break;
3275 case OP_SUQ:
3276 handleSUQ(i->asTex());
3277 break;
3278 case OP_BUFQ:
3279 handleBUFQ(i);
3280 break;
3281 case OP_PIXLD:
3282 handlePIXLD(i);
3283 break;
3284 default:
3285 break;
3286 }
3287
3288 /* Kepler+ has a special opcode to compute a new base address to be used
3289 * for indirect loads.
3290 *
3291 * Maxwell+ has an additional similar requirement for indirect
3292 * interpolation ops in frag shaders.
3293 */
3294 bool doAfetch = false;
3295 if (targ->getChipset() >= NVISA_GK104_CHIPSET &&
3296 !i->perPatch &&
3297 (i->op == OP_VFETCH || i->op == OP_EXPORT) &&
3298 i->src(0).isIndirect(0)) {
3299 doAfetch = true;
3300 }
3301 if (targ->getChipset() >= NVISA_GM107_CHIPSET &&
3302 (i->op == OP_LINTERP || i->op == OP_PINTERP) &&
3303 i->src(0).isIndirect(0)) {
3304 doAfetch = true;
3305 }
3306
3307 if (doAfetch) {
3308 Value *addr = cloneShallow(func, i->getSrc(0));
3309 Instruction *afetch = bld.mkOp1(OP_AFETCH, TYPE_U32, bld.getSSA(),
3310 i->getSrc(0));
3311 afetch->setIndirect(0, 0, i->getIndirect(0, 0));
3312 addr->reg.data.offset = 0;
3313 i->setSrc(0, addr);
3314 i->setIndirect(0, 0, afetch->getDef(0));
3315 }
3316
3317 return ret;
3318 }
3319
3320 bool
runLegalizePass(Program * prog,CGStage stage) const3321 TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const
3322 {
3323 if (stage == CG_STAGE_PRE_SSA) {
3324 NVC0LoweringPass pass(prog);
3325 return pass.run(prog, false, true);
3326 } else
3327 if (stage == CG_STAGE_POST_RA) {
3328 NVC0LegalizePostRA pass(prog);
3329 return pass.run(prog, false, true);
3330 } else
3331 if (stage == CG_STAGE_SSA) {
3332 NVC0LegalizeSSA pass;
3333 return pass.run(prog, false, true);
3334 }
3335 return false;
3336 }
3337
3338 } // namespace nv50_ir
3339