1 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "SpirvShader.hpp"
16 #include "SpirvShaderDebug.hpp"
17
18 #include "Reactor/Coroutine.hpp" // rr::Yield
19
20 #include "ShaderCore.hpp"
21
22 #include <spirv/unified1/spirv.hpp>
23
24 #include <queue>
25
26 #include <fstream>
27 #include <iostream>
28
29 namespace sw {
30
Block(InsnIterator begin,InsnIterator end)31 SpirvShader::Block::Block(InsnIterator begin, InsnIterator end)
32 : begin_(begin)
33 , end_(end)
34 {
35 // Default to a Simple, this may change later.
36 kind = Block::Simple;
37
38 // Walk the instructions to find the last two of the block.
39 InsnIterator insns[2];
40 for(auto insn : *this)
41 {
42 insns[0] = insns[1];
43 insns[1] = insn;
44 }
45
46 switch(insns[1].opcode())
47 {
48 case spv::OpBranch:
49 branchInstruction = insns[1];
50 outs.emplace(Block::ID(branchInstruction.word(1)));
51
52 switch(insns[0].opcode())
53 {
54 case spv::OpLoopMerge:
55 kind = Loop;
56 mergeInstruction = insns[0];
57 mergeBlock = Block::ID(mergeInstruction.word(1));
58 continueTarget = Block::ID(mergeInstruction.word(2));
59 break;
60
61 default:
62 kind = Block::Simple;
63 break;
64 }
65 break;
66
67 case spv::OpBranchConditional:
68 branchInstruction = insns[1];
69 outs.emplace(Block::ID(branchInstruction.word(2)));
70 outs.emplace(Block::ID(branchInstruction.word(3)));
71
72 switch(insns[0].opcode())
73 {
74 case spv::OpSelectionMerge:
75 kind = StructuredBranchConditional;
76 mergeInstruction = insns[0];
77 mergeBlock = Block::ID(mergeInstruction.word(1));
78 break;
79
80 case spv::OpLoopMerge:
81 kind = Loop;
82 mergeInstruction = insns[0];
83 mergeBlock = Block::ID(mergeInstruction.word(1));
84 continueTarget = Block::ID(mergeInstruction.word(2));
85 break;
86
87 default:
88 kind = UnstructuredBranchConditional;
89 break;
90 }
91 break;
92
93 case spv::OpSwitch:
94 branchInstruction = insns[1];
95 outs.emplace(Block::ID(branchInstruction.word(2)));
96 for(uint32_t w = 4; w < branchInstruction.wordCount(); w += 2)
97 {
98 outs.emplace(Block::ID(branchInstruction.word(w)));
99 }
100
101 switch(insns[0].opcode())
102 {
103 case spv::OpSelectionMerge:
104 kind = StructuredSwitch;
105 mergeInstruction = insns[0];
106 mergeBlock = Block::ID(mergeInstruction.word(1));
107 break;
108
109 default:
110 kind = UnstructuredSwitch;
111 break;
112 }
113 break;
114
115 default:
116 break;
117 }
118 }
119
TraverseReachableBlocks(Block::ID id,SpirvShader::Block::Set & reachable) const120 void SpirvShader::Function::TraverseReachableBlocks(Block::ID id, SpirvShader::Block::Set &reachable) const
121 {
122 if(reachable.count(id) == 0)
123 {
124 reachable.emplace(id);
125 for(auto out : getBlock(id).outs)
126 {
127 TraverseReachableBlocks(out, reachable);
128 }
129 }
130 }
131
AssignBlockFields()132 void SpirvShader::Function::AssignBlockFields()
133 {
134 Block::Set reachable;
135 TraverseReachableBlocks(entry, reachable);
136
137 for(auto &it : blocks)
138 {
139 auto &blockId = it.first;
140 auto &block = it.second;
141 if(reachable.count(blockId) > 0)
142 {
143 for(auto &outId : it.second.outs)
144 {
145 auto outIt = blocks.find(outId);
146 ASSERT_MSG(outIt != blocks.end(), "Block %d has a non-existent out %d", blockId.value(), outId.value());
147 auto &out = outIt->second;
148 out.ins.emplace(blockId);
149 }
150 if(block.kind == Block::Loop)
151 {
152 auto mergeIt = blocks.find(block.mergeBlock);
153 ASSERT_MSG(mergeIt != blocks.end(), "Loop block %d has a non-existent merge block %d", blockId.value(), block.mergeBlock.value());
154 mergeIt->second.isLoopMerge = true;
155 }
156 }
157 }
158 }
159
ForeachBlockDependency(Block::ID blockId,std::function<void (Block::ID)> f) const160 void SpirvShader::Function::ForeachBlockDependency(Block::ID blockId, std::function<void(Block::ID)> f) const
161 {
162 auto block = getBlock(blockId);
163 for(auto dep : block.ins)
164 {
165 if(block.kind != Block::Loop || // if not a loop...
166 !ExistsPath(blockId, dep, block.mergeBlock)) // or a loop and not a loop back edge
167 {
168 f(dep);
169 }
170 }
171 }
172
ExistsPath(Block::ID from,Block::ID to,Block::ID notPassingThrough) const173 bool SpirvShader::Function::ExistsPath(Block::ID from, Block::ID to, Block::ID notPassingThrough) const
174 {
175 // TODO: Optimize: This can be cached on the block.
176 Block::Set seen;
177 seen.emplace(notPassingThrough);
178
179 std::queue<Block::ID> pending;
180 pending.emplace(from);
181
182 while(pending.size() > 0)
183 {
184 auto id = pending.front();
185 pending.pop();
186 for(auto out : getBlock(id).outs)
187 {
188 if(seen.count(out) != 0) { continue; }
189 if(out == to) { return true; }
190 pending.emplace(out);
191 }
192 seen.emplace(id);
193 }
194
195 return false;
196 }
197
addOutputActiveLaneMaskEdge(Block::ID to,RValue<SIMD::Int> mask)198 void SpirvShader::EmitState::addOutputActiveLaneMaskEdge(Block::ID to, RValue<SIMD::Int> mask)
199 {
200 addActiveLaneMaskEdge(block, to, mask & activeLaneMask());
201 }
202
addActiveLaneMaskEdge(Block::ID from,Block::ID to,RValue<SIMD::Int> mask)203 void SpirvShader::EmitState::addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask)
204 {
205 auto edge = Block::Edge{ from, to };
206 auto it = edgeActiveLaneMasks.find(edge);
207 if(it == edgeActiveLaneMasks.end())
208 {
209 edgeActiveLaneMasks.emplace(edge, mask);
210 }
211 else
212 {
213 auto combined = it->second | mask;
214 edgeActiveLaneMasks.erase(edge);
215 edgeActiveLaneMasks.emplace(edge, combined);
216 }
217 }
218
GetActiveLaneMaskEdge(EmitState * state,Block::ID from,Block::ID to) const219 RValue<SIMD::Int> SpirvShader::GetActiveLaneMaskEdge(EmitState *state, Block::ID from, Block::ID to) const
220 {
221 auto edge = Block::Edge{ from, to };
222 auto it = state->edgeActiveLaneMasks.find(edge);
223 ASSERT_MSG(it != state->edgeActiveLaneMasks.end(), "Could not find edge %d -> %d", from.value(), to.value());
224 return it->second;
225 }
226
EmitBlocks(Block::ID id,EmitState * state,Block::ID ignore) const227 void SpirvShader::EmitBlocks(Block::ID id, EmitState *state, Block::ID ignore /* = 0 */) const
228 {
229 auto oldPending = state->pending;
230 auto &function = getFunction(state->function);
231
232 std::deque<Block::ID> pending;
233 state->pending = &pending;
234 pending.push_front(id);
235 while(pending.size() > 0)
236 {
237 auto id = pending.front();
238
239 auto const &block = function.getBlock(id);
240 if(id == ignore)
241 {
242 pending.pop_front();
243 continue;
244 }
245
246 // Ensure all dependency blocks have been generated.
247 auto depsDone = true;
248 function.ForeachBlockDependency(id, [&](Block::ID dep) {
249 if(state->visited.count(dep) == 0)
250 {
251 state->pending->push_front(dep);
252 depsDone = false;
253 }
254 });
255
256 if(!depsDone)
257 {
258 continue;
259 }
260
261 pending.pop_front();
262
263 state->block = id;
264
265 switch(block.kind)
266 {
267 case Block::Simple:
268 case Block::StructuredBranchConditional:
269 case Block::UnstructuredBranchConditional:
270 case Block::StructuredSwitch:
271 case Block::UnstructuredSwitch:
272 EmitNonLoop(state);
273 break;
274
275 case Block::Loop:
276 EmitLoop(state);
277 break;
278
279 default:
280 UNREACHABLE("Unexpected Block Kind: %d", int(block.kind));
281 }
282 }
283
284 state->pending = oldPending;
285 }
286
EmitNonLoop(EmitState * state) const287 void SpirvShader::EmitNonLoop(EmitState *state) const
288 {
289 auto &function = getFunction(state->function);
290 auto blockId = state->block;
291 auto block = function.getBlock(blockId);
292
293 if(!state->visited.emplace(blockId).second)
294 {
295 return; // Already generated this block.
296 }
297
298 if(blockId != function.entry)
299 {
300 // Set the activeLaneMask.
301 SIMD::Int activeLaneMask(0);
302 for(auto in : block.ins)
303 {
304 auto inMask = GetActiveLaneMaskEdge(state, in, blockId);
305 SPIRV_SHADER_DBG("Block {0} -> {1} mask: {2}", in, blockId, inMask);
306 activeLaneMask |= inMask;
307 }
308 SPIRV_SHADER_DBG("Block {0} mask: {1}", blockId, activeLaneMask);
309 SetActiveLaneMask(activeLaneMask, state);
310 }
311
312 EmitInstructions(block.begin(), block.end(), state);
313
314 for(auto out : block.outs)
315 {
316 if(state->visited.count(out) == 0)
317 {
318 state->pending->push_back(out);
319 }
320 }
321
322 SPIRV_SHADER_DBG("Block {0} done", blockId);
323 }
324
EmitLoop(EmitState * state) const325 void SpirvShader::EmitLoop(EmitState *state) const
326 {
327 auto &function = getFunction(state->function);
328 auto blockId = state->block;
329 auto &block = function.getBlock(blockId);
330 auto mergeBlockId = block.mergeBlock;
331 auto &mergeBlock = function.getBlock(mergeBlockId);
332
333 if(!state->visited.emplace(blockId).second)
334 {
335 return; // Already emitted this loop.
336 }
337
338 SPIRV_SHADER_DBG("*** LOOP HEADER ***");
339
340 // Gather all the blocks that make up the loop.
341 std::unordered_set<Block::ID> loopBlocks;
342 loopBlocks.emplace(block.mergeBlock); // Stop traversal at mergeBlock.
343 function.TraverseReachableBlocks(blockId, loopBlocks);
344
345 // incomingBlocks are block ins that are not back-edges.
346 std::unordered_set<Block::ID> incomingBlocks;
347 for(auto in : block.ins)
348 {
349 if(loopBlocks.count(in) == 0)
350 {
351 incomingBlocks.emplace(in);
352 }
353 }
354
355 // Emit the loop phi instructions, and initialize them with a value from
356 // the incoming blocks.
357 for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
358 {
359 if(insn.opcode() == spv::OpPhi)
360 {
361 StorePhi(blockId, insn, state, incomingBlocks);
362 }
363 }
364
365 // loopActiveLaneMask is the mask of lanes that are continuing to loop.
366 // This is initialized with the incoming active lane masks.
367 SIMD::Int loopActiveLaneMask = SIMD::Int(0);
368 for(auto in : incomingBlocks)
369 {
370 loopActiveLaneMask |= GetActiveLaneMaskEdge(state, in, blockId);
371 }
372
373 // mergeActiveLaneMasks contains edge lane masks for the merge block.
374 // This is the union of all edge masks across all iterations of the loop.
375 std::unordered_map<Block::ID, SIMD::Int> mergeActiveLaneMasks;
376 for(auto in : function.getBlock(mergeBlockId).ins)
377 {
378 mergeActiveLaneMasks.emplace(in, SIMD::Int(0));
379 }
380
381 // Create the loop basic blocks
382 auto headerBasicBlock = Nucleus::createBasicBlock();
383 auto mergeBasicBlock = Nucleus::createBasicBlock();
384
385 // Start emitting code inside the loop.
386 Nucleus::createBr(headerBasicBlock);
387 Nucleus::setInsertBlock(headerBasicBlock);
388
389 SPIRV_SHADER_DBG("*** LOOP START (mask: {0}) ***", loopActiveLaneMask);
390
391 // Load the active lane mask.
392 SetActiveLaneMask(loopActiveLaneMask, state);
393
394 // Emit the non-phi loop header block's instructions.
395 for(auto insn = block.begin(); insn != block.end(); insn++)
396 {
397 if(insn.opcode() == spv::OpPhi)
398 {
399 LoadPhi(insn, state);
400 dbgEndEmitInstruction(insn, state);
401 }
402 else
403 {
404 EmitInstruction(insn, state);
405 }
406 }
407
408 // Emit all blocks between the loop header and the merge block, but
409 // don't emit the merge block yet.
410 for(auto out : block.outs)
411 {
412 EmitBlocks(out, state, mergeBlockId);
413 }
414
415 // Restore current block id after emitting loop blocks.
416 state->block = blockId;
417
418 // Rebuild the loopActiveLaneMask from the loop back edges.
419 loopActiveLaneMask = SIMD::Int(0);
420 for(auto in : block.ins)
421 {
422 if(function.ExistsPath(blockId, in, mergeBlockId))
423 {
424 loopActiveLaneMask |= GetActiveLaneMaskEdge(state, in, blockId);
425 }
426 }
427
428 // Add active lanes to the merge lane mask.
429 for(auto in : function.getBlock(mergeBlockId).ins)
430 {
431 auto edge = Block::Edge{ in, mergeBlockId };
432 auto it = state->edgeActiveLaneMasks.find(edge);
433 if(it != state->edgeActiveLaneMasks.end())
434 {
435 mergeActiveLaneMasks[in] |= it->second;
436 }
437 }
438
439 // Update loop phi values.
440 for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
441 {
442 if(insn.opcode() == spv::OpPhi)
443 {
444 StorePhi(blockId, insn, state, loopBlocks);
445 }
446 }
447
448 SPIRV_SHADER_DBG("*** LOOP END (mask: {0}) ***", loopActiveLaneMask);
449
450 // Use the [loop -> merge] active lane masks to update the phi values in
451 // the merge block. We need to do this to handle divergent control flow
452 // in the loop.
453 //
454 // Consider the following:
455 //
456 // int phi_source = 0;
457 // for(uint i = 0; i < 4; i++)
458 // {
459 // phi_source = 0;
460 // if(gl_GlobalInvocationID.x % 4 == i) // divergent control flow
461 // {
462 // phi_source = 42; // single lane assignment.
463 // break; // activeLaneMask for [loop->merge] is active for a single lane.
464 // }
465 // // -- we are here --
466 // }
467 // // merge block
468 // int phi = phi_source; // OpPhi
469 //
470 // In this example, with each iteration of the loop, phi_source will
471 // only have a single lane assigned. However the 'phi' value in the merge
472 // block needs to be assigned the union of all the per-lane assignments
473 // of phi_source when that lane exited the loop.
474 for(auto insn = mergeBlock.begin(); insn != mergeBlock.end(); insn++)
475 {
476 if(insn.opcode() == spv::OpPhi)
477 {
478 StorePhi(mergeBlockId, insn, state, loopBlocks);
479 }
480 }
481
482 // Loop body now done.
483 // If any lanes are still active, jump back to the loop header,
484 // otherwise jump to the merge block.
485 Nucleus::createCondBr(AnyTrue(loopActiveLaneMask).value(), headerBasicBlock, mergeBasicBlock);
486
487 // Continue emitting from the merge block.
488 Nucleus::setInsertBlock(mergeBasicBlock);
489 state->pending->push_back(mergeBlockId);
490 for(const auto &it : mergeActiveLaneMasks)
491 {
492 state->addActiveLaneMaskEdge(it.first, mergeBlockId, it.second);
493 }
494 }
495
EmitBranch(InsnIterator insn,EmitState * state) const496 SpirvShader::EmitResult SpirvShader::EmitBranch(InsnIterator insn, EmitState *state) const
497 {
498 auto target = Block::ID(insn.word(1));
499 state->addActiveLaneMaskEdge(state->block, target, state->activeLaneMask());
500 return EmitResult::Terminator;
501 }
502
EmitBranchConditional(InsnIterator insn,EmitState * state) const503 SpirvShader::EmitResult SpirvShader::EmitBranchConditional(InsnIterator insn, EmitState *state) const
504 {
505 auto &function = getFunction(state->function);
506 auto block = function.getBlock(state->block);
507 ASSERT(block.branchInstruction == insn);
508
509 auto condId = Object::ID(block.branchInstruction.word(1));
510 auto trueBlockId = Block::ID(block.branchInstruction.word(2));
511 auto falseBlockId = Block::ID(block.branchInstruction.word(3));
512
513 auto cond = Operand(this, state, condId);
514 ASSERT_MSG(getType(getObject(condId)).componentCount == 1, "Condition must be a Boolean type scalar");
515
516 // TODO: Optimize for case where all lanes take same path.
517
518 state->addOutputActiveLaneMaskEdge(trueBlockId, cond.Int(0));
519 state->addOutputActiveLaneMaskEdge(falseBlockId, ~cond.Int(0));
520
521 return EmitResult::Terminator;
522 }
523
EmitSwitch(InsnIterator insn,EmitState * state) const524 SpirvShader::EmitResult SpirvShader::EmitSwitch(InsnIterator insn, EmitState *state) const
525 {
526 auto &function = getFunction(state->function);
527 auto block = function.getBlock(state->block);
528 ASSERT(block.branchInstruction == insn);
529
530 auto selId = Object::ID(block.branchInstruction.word(1));
531
532 auto sel = Operand(this, state, selId);
533 ASSERT_MSG(sel.componentCount == 1, "Selector must be a scalar");
534 SPIRV_SHADER_DBG("switch({0})", sel);
535
536 auto numCases = (block.branchInstruction.wordCount() - 3) / 2;
537
538 // TODO: Optimize for case where all lanes take same path.
539
540 SIMD::Int defaultLaneMask = state->activeLaneMask();
541
542 // Gather up the case label matches and calculate defaultLaneMask.
543 std::vector<RValue<SIMD::Int>> caseLabelMatches;
544 caseLabelMatches.reserve(numCases);
545 for(uint32_t i = 0; i < numCases; i++)
546 {
547 auto label = block.branchInstruction.word(i * 2 + 3);
548 auto caseBlockId = Block::ID(block.branchInstruction.word(i * 2 + 4));
549 auto caseLabelMatch = CmpEQ(sel.Int(0), SIMD::Int(label));
550 SPIRV_SHADER_DBG("case {0}: {1}", label, caseLabelMatch & state->activeLaneMask());
551 state->addOutputActiveLaneMaskEdge(caseBlockId, caseLabelMatch);
552 defaultLaneMask &= ~caseLabelMatch;
553 }
554
555 auto defaultBlockId = Block::ID(block.branchInstruction.word(2));
556 SPIRV_SHADER_DBG("default: {0}", defaultLaneMask);
557 state->addOutputActiveLaneMaskEdge(defaultBlockId, defaultLaneMask);
558
559 return EmitResult::Terminator;
560 }
561
EmitUnreachable(InsnIterator insn,EmitState * state) const562 SpirvShader::EmitResult SpirvShader::EmitUnreachable(InsnIterator insn, EmitState *state) const
563 {
564 // TODO: Log something in this case?
565 SetActiveLaneMask(SIMD::Int(0), state);
566 return EmitResult::Terminator;
567 }
568
EmitReturn(InsnIterator insn,EmitState * state) const569 SpirvShader::EmitResult SpirvShader::EmitReturn(InsnIterator insn, EmitState *state) const
570 {
571 SetActiveLaneMask(SIMD::Int(0), state);
572 return EmitResult::Terminator;
573 }
574
EmitKill(InsnIterator insn,EmitState * state) const575 SpirvShader::EmitResult SpirvShader::EmitKill(InsnIterator insn, EmitState *state) const
576 {
577 state->routine->killMask |= SignMask(state->activeLaneMask());
578 SetActiveLaneMask(SIMD::Int(0), state);
579 return EmitResult::Terminator;
580 }
581
EmitFunctionCall(InsnIterator insn,EmitState * state) const582 SpirvShader::EmitResult SpirvShader::EmitFunctionCall(InsnIterator insn, EmitState *state) const
583 {
584 auto functionId = Function::ID(insn.word(3));
585 const auto &functionIt = functions.find(functionId);
586 ASSERT(functionIt != functions.end());
587 auto &function = functionIt->second;
588
589 // TODO(b/141246700): Add full support for spv::OpFunctionCall
590 // The only supported function is a single OpKill wrapped in a
591 // function, as a result of the "wrap OpKill" SPIRV-Tools pass
592 ASSERT(function.blocks.size() == 1);
593 spv::Op wrapOpKill[] = { spv::OpLabel, spv::OpKill };
594
595 for(const auto &block : function.blocks)
596 {
597 int insnNumber = 0;
598 for(auto blockInsn : block.second)
599 {
600 if(insnNumber > 1)
601 {
602 UNIMPLEMENTED("b/141246700: Function block number of instructions: %d", insnNumber); // FIXME(b/141246700)
603 return EmitResult::Continue;
604 }
605
606 if(blockInsn.opcode() != wrapOpKill[insnNumber++])
607 {
608 UNIMPLEMENTED("b/141246700: Function block instruction %d : %s", insnNumber - 1, OpcodeName(blockInsn.opcode())); // FIXME(b/141246700)
609 return EmitResult::Continue;
610 }
611
612 if(blockInsn.opcode() == spv::OpKill)
613 {
614 EmitInstruction(blockInsn, state);
615 }
616 }
617 }
618
619 return EmitResult::Continue;
620 }
621
EmitControlBarrier(InsnIterator insn,EmitState * state) const622 SpirvShader::EmitResult SpirvShader::EmitControlBarrier(InsnIterator insn, EmitState *state) const
623 {
624 auto executionScope = spv::Scope(GetConstScalarInt(insn.word(1)));
625 auto semantics = spv::MemorySemanticsMask(GetConstScalarInt(insn.word(3)));
626 // TODO: We probably want to consider the memory scope here. For now,
627 // just always emit the full fence.
628 Fence(semantics);
629
630 switch(executionScope)
631 {
632 case spv::ScopeWorkgroup:
633 Yield(YieldResult::ControlBarrier);
634 break;
635 case spv::ScopeSubgroup:
636 break;
637 default:
638 // See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module.
639 UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup");
640 break;
641 }
642
643 return EmitResult::Continue;
644 }
645
EmitPhi(InsnIterator insn,EmitState * state) const646 SpirvShader::EmitResult SpirvShader::EmitPhi(InsnIterator insn, EmitState *state) const
647 {
648 auto &function = getFunction(state->function);
649 auto currentBlock = function.getBlock(state->block);
650 if(!currentBlock.isLoopMerge)
651 {
652 // If this is a loop merge block, then don't attempt to update the
653 // phi values from the ins. EmitLoop() has had to take special care
654 // of this phi in order to correctly deal with divergent lanes.
655 StorePhi(state->block, insn, state, currentBlock.ins);
656 }
657 LoadPhi(insn, state);
658 return EmitResult::Continue;
659 }
660
LoadPhi(InsnIterator insn,EmitState * state) const661 void SpirvShader::LoadPhi(InsnIterator insn, EmitState *state) const
662 {
663 auto typeId = Type::ID(insn.word(1));
664 auto type = getType(typeId);
665 auto objectId = Object::ID(insn.word(2));
666
667 auto storageIt = state->routine->phis.find(objectId);
668 ASSERT(storageIt != state->routine->phis.end());
669 auto &storage = storageIt->second;
670
671 auto &dst = state->createIntermediate(objectId, type.componentCount);
672 for(uint32_t i = 0; i < type.componentCount; i++)
673 {
674 dst.move(i, storage[i]);
675 SPIRV_SHADER_DBG("LoadPhi({0}.{1}): {2}", objectId, i, storage[i]);
676 }
677 }
678
StorePhi(Block::ID currentBlock,InsnIterator insn,EmitState * state,std::unordered_set<SpirvShader::Block::ID> const & filter) const679 void SpirvShader::StorePhi(Block::ID currentBlock, InsnIterator insn, EmitState *state, std::unordered_set<SpirvShader::Block::ID> const &filter) const
680 {
681 auto typeId = Type::ID(insn.word(1));
682 auto type = getType(typeId);
683 auto objectId = Object::ID(insn.word(2));
684
685 auto storageIt = state->routine->phis.find(objectId);
686 ASSERT(storageIt != state->routine->phis.end());
687 auto &storage = storageIt->second;
688
689 for(uint32_t w = 3; w < insn.wordCount(); w += 2)
690 {
691 auto varId = Object::ID(insn.word(w + 0));
692 auto blockId = Block::ID(insn.word(w + 1));
693
694 if(filter.count(blockId) == 0)
695 {
696 continue;
697 }
698
699 auto mask = GetActiveLaneMaskEdge(state, blockId, currentBlock);
700 auto in = Operand(this, state, varId);
701
702 for(uint32_t i = 0; i < type.componentCount; i++)
703 {
704 storage[i] = As<SIMD::Float>((As<SIMD::Int>(storage[i]) & ~mask) | (in.Int(i) & mask));
705 SPIRV_SHADER_DBG("StorePhi({0}.{1}): [{2} <- {3}] {4}: {5}, mask: {6}",
706 objectId, i, currentBlock, blockId, varId, in.UInt(i), mask);
707 }
708 }
709
710 for(uint32_t i = 0; i < type.componentCount; i++)
711 {
712 SPIRV_SHADER_DBG("StorePhi({0}.{1}): {2}", objectId, i, As<SIMD::UInt>(storage[i]));
713 }
714 }
715
Fence(spv::MemorySemanticsMask semantics) const716 void SpirvShader::Fence(spv::MemorySemanticsMask semantics) const
717 {
718 if(semantics == spv::MemorySemanticsMaskNone)
719 {
720 return; //no-op
721 }
722 rr::Fence(MemoryOrder(semantics));
723 }
724
Yield(YieldResult res) const725 void SpirvShader::Yield(YieldResult res) const
726 {
727 rr::Yield(RValue<Int>(int(res)));
728 }
729
SetActiveLaneMask(RValue<SIMD::Int> mask,EmitState * state) const730 void SpirvShader::SetActiveLaneMask(RValue<SIMD::Int> mask, EmitState *state) const
731 {
732 state->activeLaneMaskValue = mask.value();
733 dbgUpdateActiveLaneMask(mask, state);
734 }
735
WriteCFGGraphVizDotFile(const char * path) const736 void SpirvShader::WriteCFGGraphVizDotFile(const char *path) const
737 {
738 std::ofstream file(path);
739 file << "digraph D {" << std::endl;
740 for(auto &func : functions)
741 {
742 file << " subgraph cluster_function_" << func.first.value() << " {"
743 << std::endl;
744
745 file << " label = \"function<" << func.first.value() << ">"
746 << (func.first == entryPoint ? " (entry point)" : "")
747 << "\"" << std::endl;
748
749 for(auto &block : func.second.blocks)
750 {
751 file << " block_" << block.first.value() << " ["
752 << "shape=circle "
753 << "label=\"" << block.first.value() << "\""
754 << "]" << std::endl;
755 }
756 file << std::endl;
757 for(auto &block : func.second.blocks)
758 {
759 file << " block_" << block.first.value() << " -> {";
760 bool first = true;
761 for(auto outs : block.second.outs)
762 {
763 if(!first) { file << ", "; }
764 file << "block_" << outs.value();
765 first = false;
766 }
767 file << "}" << std::endl;
768 }
769 file << std::endl;
770 for(auto &block : func.second.blocks)
771 {
772 if(block.second.kind == Block::Loop)
773 {
774 if(block.second.mergeBlock != 0)
775 {
776 file << " block_" << block.first.value() << " -> "
777 << "block_" << block.second.mergeBlock.value()
778 << "[label=\"M\" style=dashed color=blue]"
779 << std::endl;
780 }
781 if(block.second.continueTarget != 0)
782 {
783 file << " block_" << block.first.value() << " -> "
784 << "block_" << block.second.continueTarget.value()
785 << "[label=\"C\" style=dashed color=green]"
786 << std::endl;
787 }
788 }
789 }
790
791 file << " }" << std::endl;
792 }
793
794 for(auto &func : functions)
795 {
796 for(auto &block : func.second.blocks)
797 {
798 for(auto insn : block.second)
799 {
800 if(insn.opcode() == spv::OpFunctionCall)
801 {
802 auto target = getFunction(insn.word(3)).entry;
803 file << " block_" << block.first.value() << " -> "
804 << "block_" << target.value()
805 << "[color=\"#00008050\"]"
806 << std::endl;
807 }
808 }
809 }
810 }
811
812 file << "}" << std::endl;
813 }
814
815 } // namespace sw
816