• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- StructurizeCFG.cpp ------------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "llvm/Transforms/Scalar.h"
11 #include "llvm/ADT/MapVector.h"
12 #include "llvm/ADT/PostOrderIterator.h"
13 #include "llvm/ADT/SCCIterator.h"
14 #include "llvm/Analysis/DivergenceAnalysis.h"
15 #include "llvm/Analysis/LoopInfo.h"
16 #include "llvm/Analysis/RegionInfo.h"
17 #include "llvm/Analysis/RegionIterator.h"
18 #include "llvm/Analysis/RegionPass.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/PatternMatch.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/raw_ostream.h"
23 #include "llvm/Transforms/Utils/SSAUpdater.h"
24 
25 using namespace llvm;
26 using namespace llvm::PatternMatch;
27 
28 #define DEBUG_TYPE "structurizecfg"
29 
30 namespace {
31 
32 // Definition of the complex types used in this pass.
33 
34 typedef std::pair<BasicBlock *, Value *> BBValuePair;
35 
36 typedef SmallVector<RegionNode*, 8> RNVector;
37 typedef SmallVector<BasicBlock*, 8> BBVector;
38 typedef SmallVector<BranchInst*, 8> BranchVector;
39 typedef SmallVector<BBValuePair, 2> BBValueVector;
40 
41 typedef SmallPtrSet<BasicBlock *, 8> BBSet;
42 
43 typedef MapVector<PHINode *, BBValueVector> PhiMap;
44 typedef MapVector<BasicBlock *, BBVector> BB2BBVecMap;
45 
46 typedef DenseMap<DomTreeNode *, unsigned> DTN2UnsignedMap;
47 typedef DenseMap<BasicBlock *, PhiMap> BBPhiMap;
48 typedef DenseMap<BasicBlock *, Value *> BBPredicates;
49 typedef DenseMap<BasicBlock *, BBPredicates> PredMap;
50 typedef DenseMap<BasicBlock *, BasicBlock*> BB2BBMap;
51 
52 // The name for newly created blocks.
53 
54 static const char *const FlowBlockName = "Flow";
55 
56 /// @brief Find the nearest common dominator for multiple BasicBlocks
57 ///
58 /// Helper class for StructurizeCFG
59 /// TODO: Maybe move into common code
60 class NearestCommonDominator {
61   DominatorTree *DT;
62 
63   DTN2UnsignedMap IndexMap;
64 
65   BasicBlock *Result;
66   unsigned ResultIndex;
67   bool ExplicitMentioned;
68 
69 public:
70   /// \brief Start a new query
NearestCommonDominator(DominatorTree * DomTree)71   NearestCommonDominator(DominatorTree *DomTree) {
72     DT = DomTree;
73     Result = nullptr;
74   }
75 
76   /// \brief Add BB to the resulting dominator
addBlock(BasicBlock * BB,bool Remember=true)77   void addBlock(BasicBlock *BB, bool Remember = true) {
78     DomTreeNode *Node = DT->getNode(BB);
79 
80     if (!Result) {
81       unsigned Numbering = 0;
82       for (;Node;Node = Node->getIDom())
83         IndexMap[Node] = ++Numbering;
84       Result = BB;
85       ResultIndex = 1;
86       ExplicitMentioned = Remember;
87       return;
88     }
89 
90     for (;Node;Node = Node->getIDom())
91       if (IndexMap.count(Node))
92         break;
93       else
94         IndexMap[Node] = 0;
95 
96     assert(Node && "Dominator tree invalid!");
97 
98     unsigned Numbering = IndexMap[Node];
99     if (Numbering > ResultIndex) {
100       Result = Node->getBlock();
101       ResultIndex = Numbering;
102       ExplicitMentioned = Remember && (Result == BB);
103     } else if (Numbering == ResultIndex) {
104       ExplicitMentioned |= Remember;
105     }
106   }
107 
108   /// \brief Is "Result" one of the BBs added with "Remember" = True?
wasResultExplicitMentioned()109   bool wasResultExplicitMentioned() {
110     return ExplicitMentioned;
111   }
112 
113   /// \brief Get the query result
getResult()114   BasicBlock *getResult() {
115     return Result;
116   }
117 };
118 
119 /// @brief Transforms the control flow graph on one single entry/exit region
120 /// at a time.
121 ///
122 /// After the transform all "If"/"Then"/"Else" style control flow looks like
123 /// this:
124 ///
125 /// \verbatim
126 /// 1
127 /// ||
128 /// | |
129 /// 2 |
130 /// | /
131 /// |/
132 /// 3
133 /// ||   Where:
134 /// | |  1 = "If" block, calculates the condition
135 /// 4 |  2 = "Then" subregion, runs if the condition is true
136 /// | /  3 = "Flow" blocks, newly inserted flow blocks, rejoins the flow
137 /// |/   4 = "Else" optional subregion, runs if the condition is false
138 /// 5    5 = "End" block, also rejoins the control flow
139 /// \endverbatim
140 ///
141 /// Control flow is expressed as a branch where the true exit goes into the
142 /// "Then"/"Else" region, while the false exit skips the region
143 /// The condition for the optional "Else" region is expressed as a PHI node.
144 /// The incomming values of the PHI node are true for the "If" edge and false
145 /// for the "Then" edge.
146 ///
147 /// Additionally to that even complicated loops look like this:
148 ///
149 /// \verbatim
150 /// 1
151 /// ||
152 /// | |
153 /// 2 ^  Where:
154 /// | /  1 = "Entry" block
155 /// |/   2 = "Loop" optional subregion, with all exits at "Flow" block
156 /// 3    3 = "Flow" block, with back edge to entry block
157 /// |
158 /// \endverbatim
159 ///
160 /// The back edge of the "Flow" block is always on the false side of the branch
161 /// while the true side continues the general flow. So the loop condition
162 /// consist of a network of PHI nodes where the true incoming values expresses
163 /// breaks and the false values expresses continue states.
164 class StructurizeCFG : public RegionPass {
165   bool SkipUniformRegions;
166   DivergenceAnalysis *DA;
167 
168   Type *Boolean;
169   ConstantInt *BoolTrue;
170   ConstantInt *BoolFalse;
171   UndefValue *BoolUndef;
172 
173   Function *Func;
174   Region *ParentRegion;
175 
176   DominatorTree *DT;
177   LoopInfo *LI;
178 
179   RNVector Order;
180   BBSet Visited;
181 
182   BBPhiMap DeletedPhis;
183   BB2BBVecMap AddedPhis;
184 
185   PredMap Predicates;
186   BranchVector Conditions;
187 
188   BB2BBMap Loops;
189   PredMap LoopPreds;
190   BranchVector LoopConds;
191 
192   RegionNode *PrevNode;
193 
194   void orderNodes();
195 
196   void analyzeLoops(RegionNode *N);
197 
198   Value *invert(Value *Condition);
199 
200   Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
201 
202   void gatherPredicates(RegionNode *N);
203 
204   void collectInfos();
205 
206   void insertConditions(bool Loops);
207 
208   void delPhiValues(BasicBlock *From, BasicBlock *To);
209 
210   void addPhiValues(BasicBlock *From, BasicBlock *To);
211 
212   void setPhiValues();
213 
214   void killTerminator(BasicBlock *BB);
215 
216   void changeExit(RegionNode *Node, BasicBlock *NewExit,
217                   bool IncludeDominator);
218 
219   BasicBlock *getNextFlow(BasicBlock *Dominator);
220 
221   BasicBlock *needPrefix(bool NeedEmpty);
222 
223   BasicBlock *needPostfix(BasicBlock *Flow, bool ExitUseAllowed);
224 
225   void setPrevNode(BasicBlock *BB);
226 
227   bool dominatesPredicates(BasicBlock *BB, RegionNode *Node);
228 
229   bool isPredictableTrue(RegionNode *Node);
230 
231   void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
232 
233   void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
234 
235   void createFlow();
236 
237   void rebuildSSA();
238 
239   bool hasOnlyUniformBranches(const Region *R);
240 
241 public:
242   static char ID;
243 
StructurizeCFG()244   StructurizeCFG() :
245     RegionPass(ID), SkipUniformRegions(false) {
246     initializeStructurizeCFGPass(*PassRegistry::getPassRegistry());
247   }
248 
StructurizeCFG(bool SkipUniformRegions)249   StructurizeCFG(bool SkipUniformRegions) :
250     RegionPass(ID), SkipUniformRegions(SkipUniformRegions) {
251     initializeStructurizeCFGPass(*PassRegistry::getPassRegistry());
252   }
253 
254   using Pass::doInitialization;
255   bool doInitialization(Region *R, RGPassManager &RGM) override;
256 
257   bool runOnRegion(Region *R, RGPassManager &RGM) override;
258 
getPassName() const259   const char *getPassName() const override {
260     return "Structurize control flow";
261   }
262 
getAnalysisUsage(AnalysisUsage & AU) const263   void getAnalysisUsage(AnalysisUsage &AU) const override {
264     if (SkipUniformRegions)
265       AU.addRequired<DivergenceAnalysis>();
266     AU.addRequiredID(LowerSwitchID);
267     AU.addRequired<DominatorTreeWrapperPass>();
268     AU.addRequired<LoopInfoWrapperPass>();
269     AU.addPreserved<DominatorTreeWrapperPass>();
270     RegionPass::getAnalysisUsage(AU);
271   }
272 };
273 
274 } // end anonymous namespace
275 
276 char StructurizeCFG::ID = 0;
277 
278 INITIALIZE_PASS_BEGIN(StructurizeCFG, "structurizecfg", "Structurize the CFG",
279                       false, false)
INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)280 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
281 INITIALIZE_PASS_DEPENDENCY(LowerSwitch)
282 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
283 INITIALIZE_PASS_DEPENDENCY(RegionInfoPass)
284 INITIALIZE_PASS_END(StructurizeCFG, "structurizecfg", "Structurize the CFG",
285                     false, false)
286 
287 /// \brief Initialize the types and constants used in the pass
288 bool StructurizeCFG::doInitialization(Region *R, RGPassManager &RGM) {
289   LLVMContext &Context = R->getEntry()->getContext();
290 
291   Boolean = Type::getInt1Ty(Context);
292   BoolTrue = ConstantInt::getTrue(Context);
293   BoolFalse = ConstantInt::getFalse(Context);
294   BoolUndef = UndefValue::get(Boolean);
295 
296   return false;
297 }
298 
299 /// \brief Build up the general order of nodes
orderNodes()300 void StructurizeCFG::orderNodes() {
301   RNVector TempOrder;
302   ReversePostOrderTraversal<Region*> RPOT(ParentRegion);
303   TempOrder.append(RPOT.begin(), RPOT.end());
304 
305   std::map<Loop*, unsigned> LoopBlocks;
306 
307 
308   // The reverse post-order traversal of the list gives us an ordering close
309   // to what we want.  The only problem with it is that sometimes backedges
310   // for outer loops will be visited before backedges for inner loops.
311   for (RegionNode *RN : TempOrder) {
312     BasicBlock *BB = RN->getEntry();
313     Loop *Loop = LI->getLoopFor(BB);
314     ++LoopBlocks[Loop];
315   }
316 
317   unsigned CurrentLoopDepth = 0;
318   Loop *CurrentLoop = nullptr;
319   BBSet TempVisited;
320   for (RNVector::iterator I = TempOrder.begin(), E = TempOrder.end(); I != E; ++I) {
321     BasicBlock *BB = (*I)->getEntry();
322     unsigned LoopDepth = LI->getLoopDepth(BB);
323 
324     if (std::find(Order.begin(), Order.end(), *I) != Order.end())
325       continue;
326 
327     if (LoopDepth < CurrentLoopDepth) {
328       // Make sure we have visited all blocks in this loop before moving back to
329       // the outer loop.
330 
331       RNVector::iterator LoopI = I;
332       while (unsigned &BlockCount = LoopBlocks[CurrentLoop]) {
333         LoopI++;
334         BasicBlock *LoopBB = (*LoopI)->getEntry();
335         if (LI->getLoopFor(LoopBB) == CurrentLoop) {
336           --BlockCount;
337           Order.push_back(*LoopI);
338         }
339       }
340     }
341 
342     CurrentLoop = LI->getLoopFor(BB);
343     if (CurrentLoop) {
344       LoopBlocks[CurrentLoop]--;
345     }
346 
347     CurrentLoopDepth = LoopDepth;
348     Order.push_back(*I);
349   }
350 
351   // This pass originally used a post-order traversal and then operated on
352   // the list in reverse. Now that we are using a reverse post-order traversal
353   // rather than re-working the whole pass to operate on the list in order,
354   // we just reverse the list and continue to operate on it in reverse.
355   std::reverse(Order.begin(), Order.end());
356 }
357 
358 /// \brief Determine the end of the loops
analyzeLoops(RegionNode * N)359 void StructurizeCFG::analyzeLoops(RegionNode *N) {
360   if (N->isSubRegion()) {
361     // Test for exit as back edge
362     BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
363     if (Visited.count(Exit))
364       Loops[Exit] = N->getEntry();
365 
366   } else {
367     // Test for sucessors as back edge
368     BasicBlock *BB = N->getNodeAs<BasicBlock>();
369     BranchInst *Term = cast<BranchInst>(BB->getTerminator());
370 
371     for (BasicBlock *Succ : Term->successors())
372       if (Visited.count(Succ))
373         Loops[Succ] = BB;
374   }
375 }
376 
377 /// \brief Invert the given condition
invert(Value * Condition)378 Value *StructurizeCFG::invert(Value *Condition) {
379   // First: Check if it's a constant
380   if (Condition == BoolTrue)
381     return BoolFalse;
382 
383   if (Condition == BoolFalse)
384     return BoolTrue;
385 
386   if (Condition == BoolUndef)
387     return BoolUndef;
388 
389   // Second: If the condition is already inverted, return the original value
390   if (match(Condition, m_Not(m_Value(Condition))))
391     return Condition;
392 
393   if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
394     // Third: Check all the users for an invert
395     BasicBlock *Parent = Inst->getParent();
396     for (User *U : Condition->users())
397       if (Instruction *I = dyn_cast<Instruction>(U))
398         if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
399           return I;
400 
401     // Last option: Create a new instruction
402     return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
403   }
404 
405   if (Argument *Arg = dyn_cast<Argument>(Condition)) {
406     BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
407     return BinaryOperator::CreateNot(Condition,
408                                      Arg->getName() + ".inv",
409                                      EntryBlock.getTerminator());
410   }
411 
412   llvm_unreachable("Unhandled condition to invert");
413 }
414 
415 /// \brief Build the condition for one edge
buildCondition(BranchInst * Term,unsigned Idx,bool Invert)416 Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
417                                       bool Invert) {
418   Value *Cond = Invert ? BoolFalse : BoolTrue;
419   if (Term->isConditional()) {
420     Cond = Term->getCondition();
421 
422     if (Idx != (unsigned)Invert)
423       Cond = invert(Cond);
424   }
425   return Cond;
426 }
427 
428 /// \brief Analyze the predecessors of each block and build up predicates
gatherPredicates(RegionNode * N)429 void StructurizeCFG::gatherPredicates(RegionNode *N) {
430   RegionInfo *RI = ParentRegion->getRegionInfo();
431   BasicBlock *BB = N->getEntry();
432   BBPredicates &Pred = Predicates[BB];
433   BBPredicates &LPred = LoopPreds[BB];
434 
435   for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
436        PI != PE; ++PI) {
437 
438     // Ignore it if it's a branch from outside into our region entry
439     if (!ParentRegion->contains(*PI))
440       continue;
441 
442     Region *R = RI->getRegionFor(*PI);
443     if (R == ParentRegion) {
444 
445       // It's a top level block in our region
446       BranchInst *Term = cast<BranchInst>((*PI)->getTerminator());
447       for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
448         BasicBlock *Succ = Term->getSuccessor(i);
449         if (Succ != BB)
450           continue;
451 
452         if (Visited.count(*PI)) {
453           // Normal forward edge
454           if (Term->isConditional()) {
455             // Try to treat it like an ELSE block
456             BasicBlock *Other = Term->getSuccessor(!i);
457             if (Visited.count(Other) && !Loops.count(Other) &&
458                 !Pred.count(Other) && !Pred.count(*PI)) {
459 
460               Pred[Other] = BoolFalse;
461               Pred[*PI] = BoolTrue;
462               continue;
463             }
464           }
465           Pred[*PI] = buildCondition(Term, i, false);
466 
467         } else {
468           // Back edge
469           LPred[*PI] = buildCondition(Term, i, true);
470         }
471       }
472 
473     } else {
474 
475       // It's an exit from a sub region
476       while (R->getParent() != ParentRegion)
477         R = R->getParent();
478 
479       // Edge from inside a subregion to its entry, ignore it
480       if (*R == *N)
481         continue;
482 
483       BasicBlock *Entry = R->getEntry();
484       if (Visited.count(Entry))
485         Pred[Entry] = BoolTrue;
486       else
487         LPred[Entry] = BoolFalse;
488     }
489   }
490 }
491 
492 /// \brief Collect various loop and predicate infos
collectInfos()493 void StructurizeCFG::collectInfos() {
494   // Reset predicate
495   Predicates.clear();
496 
497   // and loop infos
498   Loops.clear();
499   LoopPreds.clear();
500 
501   // Reset the visited nodes
502   Visited.clear();
503 
504   for (RegionNode *RN : reverse(Order)) {
505 
506     DEBUG(dbgs() << "Visiting: "
507                  << (RN->isSubRegion() ? "SubRegion with entry: " : "")
508                  << RN->getEntry()->getName() << " Loop Depth: "
509                  << LI->getLoopDepth(RN->getEntry()) << "\n");
510 
511     // Analyze all the conditions leading to a node
512     gatherPredicates(RN);
513 
514     // Remember that we've seen this node
515     Visited.insert(RN->getEntry());
516 
517     // Find the last back edges
518     analyzeLoops(RN);
519   }
520 }
521 
522 /// \brief Insert the missing branch conditions
insertConditions(bool Loops)523 void StructurizeCFG::insertConditions(bool Loops) {
524   BranchVector &Conds = Loops ? LoopConds : Conditions;
525   Value *Default = Loops ? BoolTrue : BoolFalse;
526   SSAUpdater PhiInserter;
527 
528   for (BranchInst *Term : Conds) {
529     assert(Term->isConditional());
530 
531     BasicBlock *Parent = Term->getParent();
532     BasicBlock *SuccTrue = Term->getSuccessor(0);
533     BasicBlock *SuccFalse = Term->getSuccessor(1);
534 
535     PhiInserter.Initialize(Boolean, "");
536     PhiInserter.AddAvailableValue(&Func->getEntryBlock(), Default);
537     PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
538 
539     BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
540 
541     NearestCommonDominator Dominator(DT);
542     Dominator.addBlock(Parent, false);
543 
544     Value *ParentValue = nullptr;
545     for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
546          PI != PE; ++PI) {
547 
548       if (PI->first == Parent) {
549         ParentValue = PI->second;
550         break;
551       }
552       PhiInserter.AddAvailableValue(PI->first, PI->second);
553       Dominator.addBlock(PI->first);
554     }
555 
556     if (ParentValue) {
557       Term->setCondition(ParentValue);
558     } else {
559       if (!Dominator.wasResultExplicitMentioned())
560         PhiInserter.AddAvailableValue(Dominator.getResult(), Default);
561 
562       Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent));
563     }
564   }
565 }
566 
567 /// \brief Remove all PHI values coming from "From" into "To" and remember
568 /// them in DeletedPhis
delPhiValues(BasicBlock * From,BasicBlock * To)569 void StructurizeCFG::delPhiValues(BasicBlock *From, BasicBlock *To) {
570   PhiMap &Map = DeletedPhis[To];
571   for (BasicBlock::iterator I = To->begin(), E = To->end();
572        I != E && isa<PHINode>(*I);) {
573 
574     PHINode &Phi = cast<PHINode>(*I++);
575     while (Phi.getBasicBlockIndex(From) != -1) {
576       Value *Deleted = Phi.removeIncomingValue(From, false);
577       Map[&Phi].push_back(std::make_pair(From, Deleted));
578     }
579   }
580 }
581 
582 /// \brief Add a dummy PHI value as soon as we knew the new predecessor
addPhiValues(BasicBlock * From,BasicBlock * To)583 void StructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
584   for (BasicBlock::iterator I = To->begin(), E = To->end();
585        I != E && isa<PHINode>(*I);) {
586 
587     PHINode &Phi = cast<PHINode>(*I++);
588     Value *Undef = UndefValue::get(Phi.getType());
589     Phi.addIncoming(Undef, From);
590   }
591   AddedPhis[To].push_back(From);
592 }
593 
594 /// \brief Add the real PHI value as soon as everything is set up
setPhiValues()595 void StructurizeCFG::setPhiValues() {
596   SSAUpdater Updater;
597   for (const auto &AddedPhi : AddedPhis) {
598 
599     BasicBlock *To = AddedPhi.first;
600     const BBVector &From = AddedPhi.second;
601 
602     if (!DeletedPhis.count(To))
603       continue;
604 
605     PhiMap &Map = DeletedPhis[To];
606     for (const auto &PI : Map) {
607 
608       PHINode *Phi = PI.first;
609       Value *Undef = UndefValue::get(Phi->getType());
610       Updater.Initialize(Phi->getType(), "");
611       Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
612       Updater.AddAvailableValue(To, Undef);
613 
614       NearestCommonDominator Dominator(DT);
615       Dominator.addBlock(To, false);
616       for (const auto &VI : PI.second) {
617 
618         Updater.AddAvailableValue(VI.first, VI.second);
619         Dominator.addBlock(VI.first);
620       }
621 
622       if (!Dominator.wasResultExplicitMentioned())
623         Updater.AddAvailableValue(Dominator.getResult(), Undef);
624 
625       for (BasicBlock *FI : From) {
626 
627         int Idx = Phi->getBasicBlockIndex(FI);
628         assert(Idx != -1);
629         Phi->setIncomingValue(Idx, Updater.GetValueAtEndOfBlock(FI));
630       }
631     }
632 
633     DeletedPhis.erase(To);
634   }
635   assert(DeletedPhis.empty());
636 }
637 
638 /// \brief Remove phi values from all successors and then remove the terminator.
killTerminator(BasicBlock * BB)639 void StructurizeCFG::killTerminator(BasicBlock *BB) {
640   TerminatorInst *Term = BB->getTerminator();
641   if (!Term)
642     return;
643 
644   for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
645        SI != SE; ++SI) {
646 
647     delPhiValues(BB, *SI);
648   }
649 
650   Term->eraseFromParent();
651 }
652 
653 /// \brief Let node exit(s) point to NewExit
changeExit(RegionNode * Node,BasicBlock * NewExit,bool IncludeDominator)654 void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
655                                 bool IncludeDominator) {
656   if (Node->isSubRegion()) {
657     Region *SubRegion = Node->getNodeAs<Region>();
658     BasicBlock *OldExit = SubRegion->getExit();
659     BasicBlock *Dominator = nullptr;
660 
661     // Find all the edges from the sub region to the exit
662     for (pred_iterator I = pred_begin(OldExit), E = pred_end(OldExit);
663          I != E;) {
664 
665       BasicBlock *BB = *I++;
666       if (!SubRegion->contains(BB))
667         continue;
668 
669       // Modify the edges to point to the new exit
670       delPhiValues(BB, OldExit);
671       BB->getTerminator()->replaceUsesOfWith(OldExit, NewExit);
672       addPhiValues(BB, NewExit);
673 
674       // Find the new dominator (if requested)
675       if (IncludeDominator) {
676         if (!Dominator)
677           Dominator = BB;
678         else
679           Dominator = DT->findNearestCommonDominator(Dominator, BB);
680       }
681     }
682 
683     // Change the dominator (if requested)
684     if (Dominator)
685       DT->changeImmediateDominator(NewExit, Dominator);
686 
687     // Update the region info
688     SubRegion->replaceExit(NewExit);
689 
690   } else {
691     BasicBlock *BB = Node->getNodeAs<BasicBlock>();
692     killTerminator(BB);
693     BranchInst::Create(NewExit, BB);
694     addPhiValues(BB, NewExit);
695     if (IncludeDominator)
696       DT->changeImmediateDominator(NewExit, BB);
697   }
698 }
699 
700 /// \brief Create a new flow node and update dominator tree and region info
getNextFlow(BasicBlock * Dominator)701 BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
702   LLVMContext &Context = Func->getContext();
703   BasicBlock *Insert = Order.empty() ? ParentRegion->getExit() :
704                        Order.back()->getEntry();
705   BasicBlock *Flow = BasicBlock::Create(Context, FlowBlockName,
706                                         Func, Insert);
707   DT->addNewBlock(Flow, Dominator);
708   ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
709   return Flow;
710 }
711 
712 /// \brief Create a new or reuse the previous node as flow node
needPrefix(bool NeedEmpty)713 BasicBlock *StructurizeCFG::needPrefix(bool NeedEmpty) {
714   BasicBlock *Entry = PrevNode->getEntry();
715 
716   if (!PrevNode->isSubRegion()) {
717     killTerminator(Entry);
718     if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
719       return Entry;
720 
721   }
722 
723   // create a new flow node
724   BasicBlock *Flow = getNextFlow(Entry);
725 
726   // and wire it up
727   changeExit(PrevNode, Flow, true);
728   PrevNode = ParentRegion->getBBNode(Flow);
729   return Flow;
730 }
731 
732 /// \brief Returns the region exit if possible, otherwise just a new flow node
needPostfix(BasicBlock * Flow,bool ExitUseAllowed)733 BasicBlock *StructurizeCFG::needPostfix(BasicBlock *Flow,
734                                         bool ExitUseAllowed) {
735   if (Order.empty() && ExitUseAllowed) {
736     BasicBlock *Exit = ParentRegion->getExit();
737     DT->changeImmediateDominator(Exit, Flow);
738     addPhiValues(Flow, Exit);
739     return Exit;
740   }
741   return getNextFlow(Flow);
742 }
743 
744 /// \brief Set the previous node
setPrevNode(BasicBlock * BB)745 void StructurizeCFG::setPrevNode(BasicBlock *BB) {
746   PrevNode = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB)
747                                         : nullptr;
748 }
749 
750 /// \brief Does BB dominate all the predicates of Node ?
dominatesPredicates(BasicBlock * BB,RegionNode * Node)751 bool StructurizeCFG::dominatesPredicates(BasicBlock *BB, RegionNode *Node) {
752   BBPredicates &Preds = Predicates[Node->getEntry()];
753   for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
754        PI != PE; ++PI) {
755 
756     if (!DT->dominates(BB, PI->first))
757       return false;
758   }
759   return true;
760 }
761 
762 /// \brief Can we predict that this node will always be called?
isPredictableTrue(RegionNode * Node)763 bool StructurizeCFG::isPredictableTrue(RegionNode *Node) {
764   BBPredicates &Preds = Predicates[Node->getEntry()];
765   bool Dominated = false;
766 
767   // Regionentry is always true
768   if (!PrevNode)
769     return true;
770 
771   for (BBPredicates::iterator I = Preds.begin(), E = Preds.end();
772        I != E; ++I) {
773 
774     if (I->second != BoolTrue)
775       return false;
776 
777     if (!Dominated && DT->dominates(I->first, PrevNode->getEntry()))
778       Dominated = true;
779   }
780 
781   // TODO: The dominator check is too strict
782   return Dominated;
783 }
784 
785 /// Take one node from the order vector and wire it up
wireFlow(bool ExitUseAllowed,BasicBlock * LoopEnd)786 void StructurizeCFG::wireFlow(bool ExitUseAllowed,
787                               BasicBlock *LoopEnd) {
788   RegionNode *Node = Order.pop_back_val();
789   Visited.insert(Node->getEntry());
790 
791   if (isPredictableTrue(Node)) {
792     // Just a linear flow
793     if (PrevNode) {
794       changeExit(PrevNode, Node->getEntry(), true);
795     }
796     PrevNode = Node;
797 
798   } else {
799     // Insert extra prefix node (or reuse last one)
800     BasicBlock *Flow = needPrefix(false);
801 
802     // Insert extra postfix node (or use exit instead)
803     BasicBlock *Entry = Node->getEntry();
804     BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
805 
806     // let it point to entry and next block
807     Conditions.push_back(BranchInst::Create(Entry, Next, BoolUndef, Flow));
808     addPhiValues(Flow, Entry);
809     DT->changeImmediateDominator(Entry, Flow);
810 
811     PrevNode = Node;
812     while (!Order.empty() && !Visited.count(LoopEnd) &&
813            dominatesPredicates(Entry, Order.back())) {
814       handleLoops(false, LoopEnd);
815     }
816 
817     changeExit(PrevNode, Next, false);
818     setPrevNode(Next);
819   }
820 }
821 
handleLoops(bool ExitUseAllowed,BasicBlock * LoopEnd)822 void StructurizeCFG::handleLoops(bool ExitUseAllowed,
823                                  BasicBlock *LoopEnd) {
824   RegionNode *Node = Order.back();
825   BasicBlock *LoopStart = Node->getEntry();
826 
827   if (!Loops.count(LoopStart)) {
828     wireFlow(ExitUseAllowed, LoopEnd);
829     return;
830   }
831 
832   if (!isPredictableTrue(Node))
833     LoopStart = needPrefix(true);
834 
835   LoopEnd = Loops[Node->getEntry()];
836   wireFlow(false, LoopEnd);
837   while (!Visited.count(LoopEnd)) {
838     handleLoops(false, LoopEnd);
839   }
840 
841   // If the start of the loop is the entry block, we can't branch to it so
842   // insert a new dummy entry block.
843   Function *LoopFunc = LoopStart->getParent();
844   if (LoopStart == &LoopFunc->getEntryBlock()) {
845     LoopStart->setName("entry.orig");
846 
847     BasicBlock *NewEntry =
848       BasicBlock::Create(LoopStart->getContext(),
849                          "entry",
850                          LoopFunc,
851                          LoopStart);
852     BranchInst::Create(LoopStart, NewEntry);
853   }
854 
855   // Create an extra loop end node
856   LoopEnd = needPrefix(false);
857   BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
858   LoopConds.push_back(BranchInst::Create(Next, LoopStart,
859                                          BoolUndef, LoopEnd));
860   addPhiValues(LoopEnd, LoopStart);
861   setPrevNode(Next);
862 }
863 
864 /// After this function control flow looks like it should be, but
865 /// branches and PHI nodes only have undefined conditions.
createFlow()866 void StructurizeCFG::createFlow() {
867   BasicBlock *Exit = ParentRegion->getExit();
868   bool EntryDominatesExit = DT->dominates(ParentRegion->getEntry(), Exit);
869 
870   DeletedPhis.clear();
871   AddedPhis.clear();
872   Conditions.clear();
873   LoopConds.clear();
874 
875   PrevNode = nullptr;
876   Visited.clear();
877 
878   while (!Order.empty()) {
879     handleLoops(EntryDominatesExit, nullptr);
880   }
881 
882   if (PrevNode)
883     changeExit(PrevNode, Exit, EntryDominatesExit);
884   else
885     assert(EntryDominatesExit);
886 }
887 
888 /// Handle a rare case where the disintegrated nodes instructions
889 /// no longer dominate all their uses. Not sure if this is really nessasary
rebuildSSA()890 void StructurizeCFG::rebuildSSA() {
891   SSAUpdater Updater;
892   for (auto *BB : ParentRegion->blocks())
893     for (BasicBlock::iterator II = BB->begin(), IE = BB->end();
894          II != IE; ++II) {
895 
896       bool Initialized = false;
897       for (auto I = II->use_begin(), E = II->use_end(); I != E;) {
898         Use &U = *I++;
899         Instruction *User = cast<Instruction>(U.getUser());
900         if (User->getParent() == BB) {
901           continue;
902 
903         } else if (PHINode *UserPN = dyn_cast<PHINode>(User)) {
904           if (UserPN->getIncomingBlock(U) == BB)
905             continue;
906         }
907 
908         if (DT->dominates(&*II, User))
909           continue;
910 
911         if (!Initialized) {
912           Value *Undef = UndefValue::get(II->getType());
913           Updater.Initialize(II->getType(), "");
914           Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
915           Updater.AddAvailableValue(BB, &*II);
916           Initialized = true;
917         }
918         Updater.RewriteUseAfterInsertions(U);
919       }
920     }
921 }
922 
hasOnlyUniformBranches(const Region * R)923 bool StructurizeCFG::hasOnlyUniformBranches(const Region *R) {
924   for (const BasicBlock *BB : R->blocks()) {
925     const BranchInst *Br = dyn_cast<BranchInst>(BB->getTerminator());
926     if (!Br || !Br->isConditional())
927       continue;
928 
929     if (!DA->isUniform(Br->getCondition()))
930       return false;
931     DEBUG(dbgs() << "BB: " << BB->getName() << " has uniform terminator\n");
932   }
933   return true;
934 }
935 
936 /// \brief Run the transformation for each region found
runOnRegion(Region * R,RGPassManager & RGM)937 bool StructurizeCFG::runOnRegion(Region *R, RGPassManager &RGM) {
938   if (R->isTopLevelRegion())
939     return false;
940 
941   if (SkipUniformRegions) {
942     DA = &getAnalysis<DivergenceAnalysis>();
943     // TODO: We could probably be smarter here with how we handle sub-regions.
944     if (hasOnlyUniformBranches(R)) {
945       DEBUG(dbgs() << "Skipping region with uniform control flow: " << *R << '\n');
946 
947       // Mark all direct child block terminators as having been treated as
948       // uniform. To account for a possible future in which non-uniform
949       // sub-regions are treated more cleverly, indirect children are not
950       // marked as uniform.
951       MDNode *MD = MDNode::get(R->getEntry()->getParent()->getContext(), {});
952       Region::element_iterator E = R->element_end();
953       for (Region::element_iterator I = R->element_begin(); I != E; ++I) {
954         if (I->isSubRegion())
955           continue;
956 
957         if (Instruction *Term = I->getEntry()->getTerminator())
958           Term->setMetadata("structurizecfg.uniform", MD);
959       }
960 
961       return false;
962     }
963   }
964 
965   Func = R->getEntry()->getParent();
966   ParentRegion = R;
967 
968   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
969   LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
970 
971   orderNodes();
972   collectInfos();
973   createFlow();
974   insertConditions(false);
975   insertConditions(true);
976   setPhiValues();
977   rebuildSSA();
978 
979   // Cleanup
980   Order.clear();
981   Visited.clear();
982   DeletedPhis.clear();
983   AddedPhis.clear();
984   Predicates.clear();
985   Conditions.clear();
986   Loops.clear();
987   LoopPreds.clear();
988   LoopConds.clear();
989 
990   return true;
991 }
992 
createStructurizeCFGPass(bool SkipUniformRegions)993 Pass *llvm::createStructurizeCFGPass(bool SkipUniformRegions) {
994   return new StructurizeCFG(SkipUniformRegions);
995 }
996