• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===--- RDFGraph.cpp -----------------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Target-independent, SSA-based data flow graph for register data flow (RDF).
11 //
12 #include "RDFGraph.h"
13 
14 #include "llvm/ADT/SetVector.h"
15 #include "llvm/CodeGen/MachineBasicBlock.h"
16 #include "llvm/CodeGen/MachineDominanceFrontier.h"
17 #include "llvm/CodeGen/MachineDominators.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/Target/TargetInstrInfo.h"
21 #include "llvm/Target/TargetRegisterInfo.h"
22 
23 using namespace llvm;
24 using namespace rdf;
25 
26 // Printing functions. Have them here first, so that the rest of the code
27 // can use them.
28 namespace llvm {
29 namespace rdf {
30 
31 template<>
operator <<(raw_ostream & OS,const Print<RegisterRef> & P)32 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterRef> &P) {
33   auto &TRI = P.G.getTRI();
34   if (P.Obj.Reg > 0 && P.Obj.Reg < TRI.getNumRegs())
35     OS << TRI.getName(P.Obj.Reg);
36   else
37     OS << '#' << P.Obj.Reg;
38   if (P.Obj.Sub > 0) {
39     OS << ':';
40     if (P.Obj.Sub < TRI.getNumSubRegIndices())
41       OS << TRI.getSubRegIndexName(P.Obj.Sub);
42     else
43       OS << '#' << P.Obj.Sub;
44   }
45   return OS;
46 }
47 
48 template<>
operator <<(raw_ostream & OS,const Print<NodeId> & P)49 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeId> &P) {
50   auto NA = P.G.addr<NodeBase*>(P.Obj);
51   uint16_t Attrs = NA.Addr->getAttrs();
52   uint16_t Kind = NodeAttrs::kind(Attrs);
53   uint16_t Flags = NodeAttrs::flags(Attrs);
54   switch (NodeAttrs::type(Attrs)) {
55     case NodeAttrs::Code:
56       switch (Kind) {
57         case NodeAttrs::Func:   OS << 'f'; break;
58         case NodeAttrs::Block:  OS << 'b'; break;
59         case NodeAttrs::Stmt:   OS << 's'; break;
60         case NodeAttrs::Phi:    OS << 'p'; break;
61         default:                OS << "c?"; break;
62       }
63       break;
64     case NodeAttrs::Ref:
65       if (Flags & NodeAttrs::Preserving)
66         OS << '+';
67       if (Flags & NodeAttrs::Clobbering)
68         OS << '~';
69       switch (Kind) {
70         case NodeAttrs::Use:    OS << 'u'; break;
71         case NodeAttrs::Def:    OS << 'd'; break;
72         case NodeAttrs::Block:  OS << 'b'; break;
73         default:                OS << "r?"; break;
74       }
75       break;
76     default:
77       OS << '?';
78       break;
79   }
80   OS << P.Obj;
81   if (Flags & NodeAttrs::Shadow)
82     OS << '"';
83   return OS;
84 }
85 
86 namespace {
printRefHeader(raw_ostream & OS,const NodeAddr<RefNode * > RA,const DataFlowGraph & G)87   void printRefHeader(raw_ostream &OS, const NodeAddr<RefNode*> RA,
88         const DataFlowGraph &G) {
89     OS << Print<NodeId>(RA.Id, G) << '<'
90        << Print<RegisterRef>(RA.Addr->getRegRef(), G) << '>';
91     if (RA.Addr->getFlags() & NodeAttrs::Fixed)
92       OS << '!';
93   }
94 }
95 
96 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<DefNode * >> & P)97 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<DefNode*>> &P) {
98   printRefHeader(OS, P.Obj, P.G);
99   OS << '(';
100   if (NodeId N = P.Obj.Addr->getReachingDef())
101     OS << Print<NodeId>(N, P.G);
102   OS << ',';
103   if (NodeId N = P.Obj.Addr->getReachedDef())
104     OS << Print<NodeId>(N, P.G);
105   OS << ',';
106   if (NodeId N = P.Obj.Addr->getReachedUse())
107     OS << Print<NodeId>(N, P.G);
108   OS << "):";
109   if (NodeId N = P.Obj.Addr->getSibling())
110     OS << Print<NodeId>(N, P.G);
111   return OS;
112 }
113 
114 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<UseNode * >> & P)115 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<UseNode*>> &P) {
116   printRefHeader(OS, P.Obj, P.G);
117   OS << '(';
118   if (NodeId N = P.Obj.Addr->getReachingDef())
119     OS << Print<NodeId>(N, P.G);
120   OS << "):";
121   if (NodeId N = P.Obj.Addr->getSibling())
122     OS << Print<NodeId>(N, P.G);
123   return OS;
124 }
125 
126 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<PhiUseNode * >> & P)127 raw_ostream &operator<< (raw_ostream &OS,
128       const Print<NodeAddr<PhiUseNode*>> &P) {
129   printRefHeader(OS, P.Obj, P.G);
130   OS << '(';
131   if (NodeId N = P.Obj.Addr->getReachingDef())
132     OS << Print<NodeId>(N, P.G);
133   OS << ',';
134   if (NodeId N = P.Obj.Addr->getPredecessor())
135     OS << Print<NodeId>(N, P.G);
136   OS << "):";
137   if (NodeId N = P.Obj.Addr->getSibling())
138     OS << Print<NodeId>(N, P.G);
139   return OS;
140 }
141 
142 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<RefNode * >> & P)143 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<RefNode*>> &P) {
144   switch (P.Obj.Addr->getKind()) {
145     case NodeAttrs::Def:
146       OS << PrintNode<DefNode*>(P.Obj, P.G);
147       break;
148     case NodeAttrs::Use:
149       if (P.Obj.Addr->getFlags() & NodeAttrs::PhiRef)
150         OS << PrintNode<PhiUseNode*>(P.Obj, P.G);
151       else
152         OS << PrintNode<UseNode*>(P.Obj, P.G);
153       break;
154   }
155   return OS;
156 }
157 
158 template<>
operator <<(raw_ostream & OS,const Print<NodeList> & P)159 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeList> &P) {
160   unsigned N = P.Obj.size();
161   for (auto I : P.Obj) {
162     OS << Print<NodeId>(I.Id, P.G);
163     if (--N)
164       OS << ' ';
165   }
166   return OS;
167 }
168 
169 template<>
operator <<(raw_ostream & OS,const Print<NodeSet> & P)170 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeSet> &P) {
171   unsigned N = P.Obj.size();
172   for (auto I : P.Obj) {
173     OS << Print<NodeId>(I, P.G);
174     if (--N)
175       OS << ' ';
176   }
177   return OS;
178 }
179 
180 namespace {
181   template <typename T>
182   struct PrintListV {
PrintListVllvm::rdf::__anon629fdc4e0211::PrintListV183     PrintListV(const NodeList &L, const DataFlowGraph &G) : List(L), G(G) {}
184     typedef T Type;
185     const NodeList &List;
186     const DataFlowGraph &G;
187   };
188 
189   template <typename T>
operator <<(raw_ostream & OS,const PrintListV<T> & P)190   raw_ostream &operator<< (raw_ostream &OS, const PrintListV<T> &P) {
191     unsigned N = P.List.size();
192     for (NodeAddr<T> A : P.List) {
193       OS << PrintNode<T>(A, P.G);
194       if (--N)
195         OS << ", ";
196     }
197     return OS;
198   }
199 }
200 
201 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<PhiNode * >> & P)202 raw_ostream &operator<< (raw_ostream &OS, const Print<NodeAddr<PhiNode*>> &P) {
203   OS << Print<NodeId>(P.Obj.Id, P.G) << ": phi ["
204      << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
205   return OS;
206 }
207 
208 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<StmtNode * >> & P)209 raw_ostream &operator<< (raw_ostream &OS,
210       const Print<NodeAddr<StmtNode*>> &P) {
211   unsigned Opc = P.Obj.Addr->getCode()->getOpcode();
212   OS << Print<NodeId>(P.Obj.Id, P.G) << ": " << P.G.getTII().getName(Opc)
213      << " [" << PrintListV<RefNode*>(P.Obj.Addr->members(P.G), P.G) << ']';
214   return OS;
215 }
216 
217 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<InstrNode * >> & P)218 raw_ostream &operator<< (raw_ostream &OS,
219       const Print<NodeAddr<InstrNode*>> &P) {
220   switch (P.Obj.Addr->getKind()) {
221     case NodeAttrs::Phi:
222       OS << PrintNode<PhiNode*>(P.Obj, P.G);
223       break;
224     case NodeAttrs::Stmt:
225       OS << PrintNode<StmtNode*>(P.Obj, P.G);
226       break;
227     default:
228       OS << "instr? " << Print<NodeId>(P.Obj.Id, P.G);
229       break;
230   }
231   return OS;
232 }
233 
234 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<BlockNode * >> & P)235 raw_ostream &operator<< (raw_ostream &OS,
236       const Print<NodeAddr<BlockNode*>> &P) {
237   auto *BB = P.Obj.Addr->getCode();
238   unsigned NP = BB->pred_size();
239   std::vector<int> Ns;
240   auto PrintBBs = [&OS,&P] (std::vector<int> Ns) -> void {
241     unsigned N = Ns.size();
242     for (auto I : Ns) {
243       OS << "BB#" << I;
244       if (--N)
245         OS << ", ";
246     }
247   };
248 
249   OS << Print<NodeId>(P.Obj.Id, P.G) << ": === BB#" << BB->getNumber()
250      << " === preds(" << NP << "): ";
251   for (auto I : BB->predecessors())
252     Ns.push_back(I->getNumber());
253   PrintBBs(Ns);
254 
255   unsigned NS = BB->succ_size();
256   OS << "  succs(" << NS << "): ";
257   Ns.clear();
258   for (auto I : BB->successors())
259     Ns.push_back(I->getNumber());
260   PrintBBs(Ns);
261   OS << '\n';
262 
263   for (auto I : P.Obj.Addr->members(P.G))
264     OS << PrintNode<InstrNode*>(I, P.G) << '\n';
265   return OS;
266 }
267 
268 template<>
operator <<(raw_ostream & OS,const Print<NodeAddr<FuncNode * >> & P)269 raw_ostream &operator<< (raw_ostream &OS,
270       const Print<NodeAddr<FuncNode*>> &P) {
271   OS << "DFG dump:[\n" << Print<NodeId>(P.Obj.Id, P.G) << ": Function: "
272      << P.Obj.Addr->getCode()->getName() << '\n';
273   for (auto I : P.Obj.Addr->members(P.G))
274     OS << PrintNode<BlockNode*>(I, P.G) << '\n';
275   OS << "]\n";
276   return OS;
277 }
278 
279 template<>
operator <<(raw_ostream & OS,const Print<RegisterSet> & P)280 raw_ostream &operator<< (raw_ostream &OS, const Print<RegisterSet> &P) {
281   OS << '{';
282   for (auto I : P.Obj)
283     OS << ' ' << Print<RegisterRef>(I, P.G);
284   OS << " }";
285   return OS;
286 }
287 
288 template<>
operator <<(raw_ostream & OS,const Print<DataFlowGraph::DefStack> & P)289 raw_ostream &operator<< (raw_ostream &OS,
290       const Print<DataFlowGraph::DefStack> &P) {
291   for (auto I = P.Obj.top(), E = P.Obj.bottom(); I != E; ) {
292     OS << Print<NodeId>(I->Id, P.G)
293        << '<' << Print<RegisterRef>(I->Addr->getRegRef(), P.G) << '>';
294     I.down();
295     if (I != E)
296       OS << ' ';
297   }
298   return OS;
299 }
300 
301 } // namespace rdf
302 } // namespace llvm
303 
304 // Node allocation functions.
305 //
306 // Node allocator is like a slab memory allocator: it allocates blocks of
307 // memory in sizes that are multiples of the size of a node. Each block has
308 // the same size. Nodes are allocated from the currently active block, and
309 // when it becomes full, a new one is created.
310 // There is a mapping scheme between node id and its location in a block,
311 // and within that block is described in the header file.
312 //
startNewBlock()313 void NodeAllocator::startNewBlock() {
314   void *T = MemPool.Allocate(NodesPerBlock*NodeMemSize, NodeMemSize);
315   char *P = static_cast<char*>(T);
316   Blocks.push_back(P);
317   // Check if the block index is still within the allowed range, i.e. less
318   // than 2^N, where N is the number of bits in NodeId for the block index.
319   // BitsPerIndex is the number of bits per node index.
320   assert((Blocks.size() < ((size_t)1 << (8*sizeof(NodeId)-BitsPerIndex))) &&
321          "Out of bits for block index");
322   ActiveEnd = P;
323 }
324 
needNewBlock()325 bool NodeAllocator::needNewBlock() {
326   if (Blocks.empty())
327     return true;
328 
329   char *ActiveBegin = Blocks.back();
330   uint32_t Index = (ActiveEnd-ActiveBegin)/NodeMemSize;
331   return Index >= NodesPerBlock;
332 }
333 
New()334 NodeAddr<NodeBase*> NodeAllocator::New() {
335   if (needNewBlock())
336     startNewBlock();
337 
338   uint32_t ActiveB = Blocks.size()-1;
339   uint32_t Index = (ActiveEnd - Blocks[ActiveB])/NodeMemSize;
340   NodeAddr<NodeBase*> NA = { reinterpret_cast<NodeBase*>(ActiveEnd),
341                              makeId(ActiveB, Index) };
342   ActiveEnd += NodeMemSize;
343   return NA;
344 }
345 
id(const NodeBase * P) const346 NodeId NodeAllocator::id(const NodeBase *P) const {
347   uintptr_t A = reinterpret_cast<uintptr_t>(P);
348   for (unsigned i = 0, n = Blocks.size(); i != n; ++i) {
349     uintptr_t B = reinterpret_cast<uintptr_t>(Blocks[i]);
350     if (A < B || A >= B + NodesPerBlock*NodeMemSize)
351       continue;
352     uint32_t Idx = (A-B)/NodeMemSize;
353     return makeId(i, Idx);
354   }
355   llvm_unreachable("Invalid node address");
356 }
357 
clear()358 void NodeAllocator::clear() {
359   MemPool.Reset();
360   Blocks.clear();
361   ActiveEnd = nullptr;
362 }
363 
364 
365 // Insert node NA after "this" in the circular chain.
append(NodeAddr<NodeBase * > NA)366 void NodeBase::append(NodeAddr<NodeBase*> NA) {
367   NodeId Nx = Next;
368   // If NA is already "next", do nothing.
369   if (Next != NA.Id) {
370     Next = NA.Id;
371     NA.Addr->Next = Nx;
372   }
373 }
374 
375 
376 // Fundamental node manipulator functions.
377 
378 // Obtain the register reference from a reference node.
getRegRef() const379 RegisterRef RefNode::getRegRef() const {
380   assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
381   if (NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef)
382     return Ref.RR;
383   assert(Ref.Op != nullptr);
384   return { Ref.Op->getReg(), Ref.Op->getSubReg() };
385 }
386 
387 // Set the register reference in the reference node directly (for references
388 // in phi nodes).
setRegRef(RegisterRef RR)389 void RefNode::setRegRef(RegisterRef RR) {
390   assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
391   assert(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef);
392   Ref.RR = RR;
393 }
394 
395 // Set the register reference in the reference node based on a machine
396 // operand (for references in statement nodes).
setRegRef(MachineOperand * Op)397 void RefNode::setRegRef(MachineOperand *Op) {
398   assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref);
399   assert(!(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef));
400   Ref.Op = Op;
401 }
402 
403 // Get the owner of a given reference node.
getOwner(const DataFlowGraph & G)404 NodeAddr<NodeBase*> RefNode::getOwner(const DataFlowGraph &G) {
405   NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
406 
407   while (NA.Addr != this) {
408     if (NA.Addr->getType() == NodeAttrs::Code)
409       return NA;
410     NA = G.addr<NodeBase*>(NA.Addr->getNext());
411   }
412   llvm_unreachable("No owner in circular list");
413 }
414 
415 // Connect the def node to the reaching def node.
linkToDef(NodeId Self,NodeAddr<DefNode * > DA)416 void DefNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
417   Ref.RD = DA.Id;
418   Ref.Sib = DA.Addr->getReachedDef();
419   DA.Addr->setReachedDef(Self);
420 }
421 
422 // Connect the use node to the reaching def node.
linkToDef(NodeId Self,NodeAddr<DefNode * > DA)423 void UseNode::linkToDef(NodeId Self, NodeAddr<DefNode*> DA) {
424   Ref.RD = DA.Id;
425   Ref.Sib = DA.Addr->getReachedUse();
426   DA.Addr->setReachedUse(Self);
427 }
428 
429 // Get the first member of the code node.
getFirstMember(const DataFlowGraph & G) const430 NodeAddr<NodeBase*> CodeNode::getFirstMember(const DataFlowGraph &G) const {
431   if (Code.FirstM == 0)
432     return NodeAddr<NodeBase*>();
433   return G.addr<NodeBase*>(Code.FirstM);
434 }
435 
436 // Get the last member of the code node.
getLastMember(const DataFlowGraph & G) const437 NodeAddr<NodeBase*> CodeNode::getLastMember(const DataFlowGraph &G) const {
438   if (Code.LastM == 0)
439     return NodeAddr<NodeBase*>();
440   return G.addr<NodeBase*>(Code.LastM);
441 }
442 
443 // Add node NA at the end of the member list of the given code node.
addMember(NodeAddr<NodeBase * > NA,const DataFlowGraph & G)444 void CodeNode::addMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
445   auto ML = getLastMember(G);
446   if (ML.Id != 0) {
447     ML.Addr->append(NA);
448   } else {
449     Code.FirstM = NA.Id;
450     NodeId Self = G.id(this);
451     NA.Addr->setNext(Self);
452   }
453   Code.LastM = NA.Id;
454 }
455 
456 // Add node NA after member node MA in the given code node.
addMemberAfter(NodeAddr<NodeBase * > MA,NodeAddr<NodeBase * > NA,const DataFlowGraph & G)457 void CodeNode::addMemberAfter(NodeAddr<NodeBase*> MA, NodeAddr<NodeBase*> NA,
458       const DataFlowGraph &G) {
459   MA.Addr->append(NA);
460   if (Code.LastM == MA.Id)
461     Code.LastM = NA.Id;
462 }
463 
464 // Remove member node NA from the given code node.
removeMember(NodeAddr<NodeBase * > NA,const DataFlowGraph & G)465 void CodeNode::removeMember(NodeAddr<NodeBase*> NA, const DataFlowGraph &G) {
466   auto MA = getFirstMember(G);
467   assert(MA.Id != 0);
468 
469   // Special handling if the member to remove is the first member.
470   if (MA.Id == NA.Id) {
471     if (Code.LastM == MA.Id) {
472       // If it is the only member, set both first and last to 0.
473       Code.FirstM = Code.LastM = 0;
474     } else {
475       // Otherwise, advance the first member.
476       Code.FirstM = MA.Addr->getNext();
477     }
478     return;
479   }
480 
481   while (MA.Addr != this) {
482     NodeId MX = MA.Addr->getNext();
483     if (MX == NA.Id) {
484       MA.Addr->setNext(NA.Addr->getNext());
485       // If the member to remove happens to be the last one, update the
486       // LastM indicator.
487       if (Code.LastM == NA.Id)
488         Code.LastM = MA.Id;
489       return;
490     }
491     MA = G.addr<NodeBase*>(MX);
492   }
493   llvm_unreachable("No such member");
494 }
495 
496 // Return the list of all members of the code node.
members(const DataFlowGraph & G) const497 NodeList CodeNode::members(const DataFlowGraph &G) const {
498   static auto True = [] (NodeAddr<NodeBase*>) -> bool { return true; };
499   return members_if(True, G);
500 }
501 
502 // Return the owner of the given instr node.
getOwner(const DataFlowGraph & G)503 NodeAddr<NodeBase*> InstrNode::getOwner(const DataFlowGraph &G) {
504   NodeAddr<NodeBase*> NA = G.addr<NodeBase*>(getNext());
505 
506   while (NA.Addr != this) {
507     assert(NA.Addr->getType() == NodeAttrs::Code);
508     if (NA.Addr->getKind() == NodeAttrs::Block)
509       return NA;
510     NA = G.addr<NodeBase*>(NA.Addr->getNext());
511   }
512   llvm_unreachable("No owner in circular list");
513 }
514 
515 // Add the phi node PA to the given block node.
addPhi(NodeAddr<PhiNode * > PA,const DataFlowGraph & G)516 void BlockNode::addPhi(NodeAddr<PhiNode*> PA, const DataFlowGraph &G) {
517   auto M = getFirstMember(G);
518   if (M.Id == 0) {
519     addMember(PA, G);
520     return;
521   }
522 
523   assert(M.Addr->getType() == NodeAttrs::Code);
524   if (M.Addr->getKind() == NodeAttrs::Stmt) {
525     // If the first member of the block is a statement, insert the phi as
526     // the first member.
527     Code.FirstM = PA.Id;
528     PA.Addr->setNext(M.Id);
529   } else {
530     // If the first member is a phi, find the last phi, and append PA to it.
531     assert(M.Addr->getKind() == NodeAttrs::Phi);
532     NodeAddr<NodeBase*> MN = M;
533     do {
534       M = MN;
535       MN = G.addr<NodeBase*>(M.Addr->getNext());
536       assert(MN.Addr->getType() == NodeAttrs::Code);
537     } while (MN.Addr->getKind() == NodeAttrs::Phi);
538 
539     // M is the last phi.
540     addMemberAfter(M, PA, G);
541   }
542 }
543 
544 // Find the block node corresponding to the machine basic block BB in the
545 // given func node.
findBlock(const MachineBasicBlock * BB,const DataFlowGraph & G) const546 NodeAddr<BlockNode*> FuncNode::findBlock(const MachineBasicBlock *BB,
547       const DataFlowGraph &G) const {
548   auto EqBB = [BB] (NodeAddr<NodeBase*> NA) -> bool {
549     return NodeAddr<BlockNode*>(NA).Addr->getCode() == BB;
550   };
551   NodeList Ms = members_if(EqBB, G);
552   if (!Ms.empty())
553     return Ms[0];
554   return NodeAddr<BlockNode*>();
555 }
556 
557 // Get the block node for the entry block in the given function.
getEntryBlock(const DataFlowGraph & G)558 NodeAddr<BlockNode*> FuncNode::getEntryBlock(const DataFlowGraph &G) {
559   MachineBasicBlock *EntryB = &getCode()->front();
560   return findBlock(EntryB, G);
561 }
562 
563 
564 // Register aliasing information.
565 //
566 // In theory, the lane information could be used to determine register
567 // covering (and aliasing), but depending on the sub-register structure,
568 // the lane mask information may be missing. The covering information
569 // must be available for this framework to work, so relying solely on
570 // the lane data is not sufficient.
571 
572 // Determine whether RA covers RB.
covers(RegisterRef RA,RegisterRef RB) const573 bool RegisterAliasInfo::covers(RegisterRef RA, RegisterRef RB) const {
574   if (RA == RB)
575     return true;
576   if (TargetRegisterInfo::isVirtualRegister(RA.Reg)) {
577     assert(TargetRegisterInfo::isVirtualRegister(RB.Reg));
578     if (RA.Reg != RB.Reg)
579       return false;
580     if (RA.Sub == 0)
581       return true;
582     return TRI.composeSubRegIndices(RA.Sub, RB.Sub) == RA.Sub;
583   }
584 
585   assert(TargetRegisterInfo::isPhysicalRegister(RA.Reg) &&
586          TargetRegisterInfo::isPhysicalRegister(RB.Reg));
587   unsigned A = RA.Sub != 0 ? TRI.getSubReg(RA.Reg, RA.Sub) : RA.Reg;
588   unsigned B = RB.Sub != 0 ? TRI.getSubReg(RB.Reg, RB.Sub) : RB.Reg;
589   return TRI.isSubRegister(A, B);
590 }
591 
592 // Determine whether RR is covered by the set of references RRs.
covers(const RegisterSet & RRs,RegisterRef RR) const593 bool RegisterAliasInfo::covers(const RegisterSet &RRs, RegisterRef RR) const {
594   if (RRs.count(RR))
595     return true;
596 
597   // For virtual registers, we cannot accurately determine covering based
598   // on subregisters. If RR itself is not present in RRs, but it has a sub-
599   // register reference, check for the super-register alone. Otherwise,
600   // assume non-covering.
601   if (TargetRegisterInfo::isVirtualRegister(RR.Reg)) {
602     if (RR.Sub != 0)
603       return RRs.count({RR.Reg, 0});
604     return false;
605   }
606 
607   // If any super-register of RR is present, then RR is covered.
608   unsigned Reg = RR.Sub == 0 ? RR.Reg : TRI.getSubReg(RR.Reg, RR.Sub);
609   for (MCSuperRegIterator SR(Reg, &TRI); SR.isValid(); ++SR)
610     if (RRs.count({*SR, 0}))
611       return true;
612 
613   return false;
614 }
615 
616 // Get the list of references aliased to RR.
getAliasSet(RegisterRef RR) const617 std::vector<RegisterRef> RegisterAliasInfo::getAliasSet(RegisterRef RR) const {
618   // Do not include RR in the alias set. For virtual registers return an
619   // empty set.
620   std::vector<RegisterRef> AS;
621   if (TargetRegisterInfo::isVirtualRegister(RR.Reg))
622     return AS;
623   assert(TargetRegisterInfo::isPhysicalRegister(RR.Reg));
624   unsigned R = RR.Reg;
625   if (RR.Sub)
626     R = TRI.getSubReg(RR.Reg, RR.Sub);
627 
628   for (MCRegAliasIterator AI(R, &TRI, false); AI.isValid(); ++AI)
629     AS.push_back(RegisterRef({*AI, 0}));
630   return AS;
631 }
632 
633 // Check whether RA and RB are aliased.
alias(RegisterRef RA,RegisterRef RB) const634 bool RegisterAliasInfo::alias(RegisterRef RA, RegisterRef RB) const {
635   bool VirtA = TargetRegisterInfo::isVirtualRegister(RA.Reg);
636   bool VirtB = TargetRegisterInfo::isVirtualRegister(RB.Reg);
637   bool PhysA = TargetRegisterInfo::isPhysicalRegister(RA.Reg);
638   bool PhysB = TargetRegisterInfo::isPhysicalRegister(RB.Reg);
639 
640   if (VirtA != VirtB)
641     return false;
642 
643   if (VirtA) {
644     if (RA.Reg != RB.Reg)
645       return false;
646     // RA and RB refer to the same register. If any of them refer to the
647     // whole register, they must be aliased.
648     if (RA.Sub == 0 || RB.Sub == 0)
649       return true;
650     unsigned SA = TRI.getSubRegIdxSize(RA.Sub);
651     unsigned OA = TRI.getSubRegIdxOffset(RA.Sub);
652     unsigned SB = TRI.getSubRegIdxSize(RB.Sub);
653     unsigned OB = TRI.getSubRegIdxOffset(RB.Sub);
654     if (OA <= OB && OA+SA > OB)
655       return true;
656     if (OB <= OA && OB+SB > OA)
657       return true;
658     return false;
659   }
660 
661   assert(PhysA && PhysB);
662   (void)PhysA, (void)PhysB;
663   unsigned A = RA.Sub ? TRI.getSubReg(RA.Reg, RA.Sub) : RA.Reg;
664   unsigned B = RB.Sub ? TRI.getSubReg(RB.Reg, RB.Sub) : RB.Reg;
665   for (MCRegAliasIterator I(A, &TRI, true); I.isValid(); ++I)
666     if (B == *I)
667       return true;
668   return false;
669 }
670 
671 
672 // Target operand information.
673 //
674 
675 // For a given instruction, check if there are any bits of RR that can remain
676 // unchanged across this def.
isPreserving(const MachineInstr & In,unsigned OpNum) const677 bool TargetOperandInfo::isPreserving(const MachineInstr &In, unsigned OpNum)
678       const {
679   return TII.isPredicated(In);
680 }
681 
682 // Check if the definition of RR produces an unspecified value.
isClobbering(const MachineInstr & In,unsigned OpNum) const683 bool TargetOperandInfo::isClobbering(const MachineInstr &In, unsigned OpNum)
684       const {
685   if (In.isCall())
686     if (In.getOperand(OpNum).isImplicit())
687       return true;
688   return false;
689 }
690 
691 // Check if the given instruction specifically requires
isFixedReg(const MachineInstr & In,unsigned OpNum) const692 bool TargetOperandInfo::isFixedReg(const MachineInstr &In, unsigned OpNum)
693       const {
694   if (In.isCall() || In.isReturn() || In.isInlineAsm())
695     return true;
696   // Check for a tail call.
697   if (In.isBranch())
698     for (auto &O : In.operands())
699       if (O.isGlobal() || O.isSymbol())
700         return true;
701 
702   const MCInstrDesc &D = In.getDesc();
703   if (!D.getImplicitDefs() && !D.getImplicitUses())
704     return false;
705   const MachineOperand &Op = In.getOperand(OpNum);
706   // If there is a sub-register, treat the operand as non-fixed. Currently,
707   // fixed registers are those that are listed in the descriptor as implicit
708   // uses or defs, and those lists do not allow sub-registers.
709   if (Op.getSubReg() != 0)
710     return false;
711   unsigned Reg = Op.getReg();
712   const MCPhysReg *ImpR = Op.isDef() ? D.getImplicitDefs()
713                                      : D.getImplicitUses();
714   if (!ImpR)
715     return false;
716   while (*ImpR)
717     if (*ImpR++ == Reg)
718       return true;
719   return false;
720 }
721 
722 
723 //
724 // The data flow graph construction.
725 //
726 
DataFlowGraph(MachineFunction & mf,const TargetInstrInfo & tii,const TargetRegisterInfo & tri,const MachineDominatorTree & mdt,const MachineDominanceFrontier & mdf,const RegisterAliasInfo & rai,const TargetOperandInfo & toi)727 DataFlowGraph::DataFlowGraph(MachineFunction &mf, const TargetInstrInfo &tii,
728       const TargetRegisterInfo &tri, const MachineDominatorTree &mdt,
729       const MachineDominanceFrontier &mdf, const RegisterAliasInfo &rai,
730       const TargetOperandInfo &toi)
731     : TimeG("rdf"), MF(mf), TII(tii), TRI(tri), MDT(mdt), MDF(mdf), RAI(rai),
732       TOI(toi) {
733 }
734 
735 
736 // The implementation of the definition stack.
737 // Each register reference has its own definition stack. In particular,
738 // for a register references "Reg" and "Reg:subreg" will each have their
739 // own definition stacks.
740 
741 // Construct a stack iterator.
Iterator(const DataFlowGraph::DefStack & S,bool Top)742 DataFlowGraph::DefStack::Iterator::Iterator(const DataFlowGraph::DefStack &S,
743       bool Top) : DS(S) {
744   if (!Top) {
745     // Initialize to bottom.
746     Pos = 0;
747     return;
748   }
749   // Initialize to the top, i.e. top-most non-delimiter (or 0, if empty).
750   Pos = DS.Stack.size();
751   while (Pos > 0 && DS.isDelimiter(DS.Stack[Pos-1]))
752     Pos--;
753 }
754 
755 // Return the size of the stack, including block delimiters.
size() const756 unsigned DataFlowGraph::DefStack::size() const {
757   unsigned S = 0;
758   for (auto I = top(), E = bottom(); I != E; I.down())
759     S++;
760   return S;
761 }
762 
763 // Remove the top entry from the stack. Remove all intervening delimiters
764 // so that after this, the stack is either empty, or the top of the stack
765 // is a non-delimiter.
pop()766 void DataFlowGraph::DefStack::pop() {
767   assert(!empty());
768   unsigned P = nextDown(Stack.size());
769   Stack.resize(P);
770 }
771 
772 // Push a delimiter for block node N on the stack.
start_block(NodeId N)773 void DataFlowGraph::DefStack::start_block(NodeId N) {
774   assert(N != 0);
775   Stack.push_back(NodeAddr<DefNode*>(nullptr, N));
776 }
777 
778 // Remove all nodes from the top of the stack, until the delimited for
779 // block node N is encountered. Remove the delimiter as well. In effect,
780 // this will remove from the stack all definitions from block N.
clear_block(NodeId N)781 void DataFlowGraph::DefStack::clear_block(NodeId N) {
782   assert(N != 0);
783   unsigned P = Stack.size();
784   while (P > 0) {
785     bool Found = isDelimiter(Stack[P-1], N);
786     P--;
787     if (Found)
788       break;
789   }
790   // This will also remove the delimiter, if found.
791   Stack.resize(P);
792 }
793 
794 // Move the stack iterator up by one.
nextUp(unsigned P) const795 unsigned DataFlowGraph::DefStack::nextUp(unsigned P) const {
796   // Get the next valid position after P (skipping all delimiters).
797   // The input position P does not have to point to a non-delimiter.
798   unsigned SS = Stack.size();
799   bool IsDelim;
800   assert(P < SS);
801   do {
802     P++;
803     IsDelim = isDelimiter(Stack[P-1]);
804   } while (P < SS && IsDelim);
805   assert(!IsDelim);
806   return P;
807 }
808 
809 // Move the stack iterator down by one.
nextDown(unsigned P) const810 unsigned DataFlowGraph::DefStack::nextDown(unsigned P) const {
811   // Get the preceding valid position before P (skipping all delimiters).
812   // The input position P does not have to point to a non-delimiter.
813   assert(P > 0 && P <= Stack.size());
814   bool IsDelim = isDelimiter(Stack[P-1]);
815   do {
816     if (--P == 0)
817       break;
818     IsDelim = isDelimiter(Stack[P-1]);
819   } while (P > 0 && IsDelim);
820   assert(!IsDelim);
821   return P;
822 }
823 
824 // Node management functions.
825 
826 // Get the pointer to the node with the id N.
ptr(NodeId N) const827 NodeBase *DataFlowGraph::ptr(NodeId N) const {
828   if (N == 0)
829     return nullptr;
830   return Memory.ptr(N);
831 }
832 
833 // Get the id of the node at the address P.
id(const NodeBase * P) const834 NodeId DataFlowGraph::id(const NodeBase *P) const {
835   if (P == nullptr)
836     return 0;
837   return Memory.id(P);
838 }
839 
840 // Allocate a new node and set the attributes to Attrs.
newNode(uint16_t Attrs)841 NodeAddr<NodeBase*> DataFlowGraph::newNode(uint16_t Attrs) {
842   NodeAddr<NodeBase*> P = Memory.New();
843   P.Addr->init();
844   P.Addr->setAttrs(Attrs);
845   return P;
846 }
847 
848 // Make a copy of the given node B, except for the data-flow links, which
849 // are set to 0.
cloneNode(const NodeAddr<NodeBase * > B)850 NodeAddr<NodeBase*> DataFlowGraph::cloneNode(const NodeAddr<NodeBase*> B) {
851   NodeAddr<NodeBase*> NA = newNode(0);
852   memcpy(NA.Addr, B.Addr, sizeof(NodeBase));
853   // Ref nodes need to have the data-flow links reset.
854   if (NA.Addr->getType() == NodeAttrs::Ref) {
855     NodeAddr<RefNode*> RA = NA;
856     RA.Addr->setReachingDef(0);
857     RA.Addr->setSibling(0);
858     if (NA.Addr->getKind() == NodeAttrs::Def) {
859       NodeAddr<DefNode*> DA = NA;
860       DA.Addr->setReachedDef(0);
861       DA.Addr->setReachedUse(0);
862     }
863   }
864   return NA;
865 }
866 
867 
868 // Allocation routines for specific node types/kinds.
869 
newUse(NodeAddr<InstrNode * > Owner,MachineOperand & Op,uint16_t Flags)870 NodeAddr<UseNode*> DataFlowGraph::newUse(NodeAddr<InstrNode*> Owner,
871       MachineOperand &Op, uint16_t Flags) {
872   NodeAddr<UseNode*> UA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
873   UA.Addr->setRegRef(&Op);
874   return UA;
875 }
876 
newPhiUse(NodeAddr<PhiNode * > Owner,RegisterRef RR,NodeAddr<BlockNode * > PredB,uint16_t Flags)877 NodeAddr<PhiUseNode*> DataFlowGraph::newPhiUse(NodeAddr<PhiNode*> Owner,
878       RegisterRef RR, NodeAddr<BlockNode*> PredB, uint16_t Flags) {
879   NodeAddr<PhiUseNode*> PUA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags);
880   assert(Flags & NodeAttrs::PhiRef);
881   PUA.Addr->setRegRef(RR);
882   PUA.Addr->setPredecessor(PredB.Id);
883   return PUA;
884 }
885 
newDef(NodeAddr<InstrNode * > Owner,MachineOperand & Op,uint16_t Flags)886 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
887       MachineOperand &Op, uint16_t Flags) {
888   NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
889   DA.Addr->setRegRef(&Op);
890   return DA;
891 }
892 
newDef(NodeAddr<InstrNode * > Owner,RegisterRef RR,uint16_t Flags)893 NodeAddr<DefNode*> DataFlowGraph::newDef(NodeAddr<InstrNode*> Owner,
894       RegisterRef RR, uint16_t Flags) {
895   NodeAddr<DefNode*> DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags);
896   assert(Flags & NodeAttrs::PhiRef);
897   DA.Addr->setRegRef(RR);
898   return DA;
899 }
900 
newPhi(NodeAddr<BlockNode * > Owner)901 NodeAddr<PhiNode*> DataFlowGraph::newPhi(NodeAddr<BlockNode*> Owner) {
902   NodeAddr<PhiNode*> PA = newNode(NodeAttrs::Code | NodeAttrs::Phi);
903   Owner.Addr->addPhi(PA, *this);
904   return PA;
905 }
906 
newStmt(NodeAddr<BlockNode * > Owner,MachineInstr * MI)907 NodeAddr<StmtNode*> DataFlowGraph::newStmt(NodeAddr<BlockNode*> Owner,
908       MachineInstr *MI) {
909   NodeAddr<StmtNode*> SA = newNode(NodeAttrs::Code | NodeAttrs::Stmt);
910   SA.Addr->setCode(MI);
911   Owner.Addr->addMember(SA, *this);
912   return SA;
913 }
914 
newBlock(NodeAddr<FuncNode * > Owner,MachineBasicBlock * BB)915 NodeAddr<BlockNode*> DataFlowGraph::newBlock(NodeAddr<FuncNode*> Owner,
916       MachineBasicBlock *BB) {
917   NodeAddr<BlockNode*> BA = newNode(NodeAttrs::Code | NodeAttrs::Block);
918   BA.Addr->setCode(BB);
919   Owner.Addr->addMember(BA, *this);
920   return BA;
921 }
922 
newFunc(MachineFunction * MF)923 NodeAddr<FuncNode*> DataFlowGraph::newFunc(MachineFunction *MF) {
924   NodeAddr<FuncNode*> FA = newNode(NodeAttrs::Code | NodeAttrs::Func);
925   FA.Addr->setCode(MF);
926   return FA;
927 }
928 
929 // Build the data flow graph.
build(unsigned Options)930 void DataFlowGraph::build(unsigned Options) {
931   reset();
932   Func = newFunc(&MF);
933 
934   if (MF.empty())
935     return;
936 
937   for (auto &B : MF) {
938     auto BA = newBlock(Func, &B);
939     for (auto &I : B) {
940       if (I.isDebugValue())
941         continue;
942       buildStmt(BA, I);
943     }
944   }
945 
946   // Collect information about block references.
947   NodeAddr<BlockNode*> EA = Func.Addr->getEntryBlock(*this);
948   BlockRefsMap RefM;
949   buildBlockRefs(EA, RefM);
950 
951   // Add function-entry phi nodes.
952   MachineRegisterInfo &MRI = MF.getRegInfo();
953   for (auto I = MRI.livein_begin(), E = MRI.livein_end(); I != E; ++I) {
954     NodeAddr<PhiNode*> PA = newPhi(EA);
955     RegisterRef RR = { I->first, 0 };
956     uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
957     NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
958     PA.Addr->addMember(DA, *this);
959   }
960 
961   // Build a map "PhiM" which will contain, for each block, the set
962   // of references that will require phi definitions in that block.
963   BlockRefsMap PhiM;
964   auto Blocks = Func.Addr->members(*this);
965   for (NodeAddr<BlockNode*> BA : Blocks)
966     recordDefsForDF(PhiM, RefM, BA);
967   for (NodeAddr<BlockNode*> BA : Blocks)
968     buildPhis(PhiM, RefM, BA);
969 
970   // Link all the refs. This will recursively traverse the dominator tree.
971   DefStackMap DM;
972   linkBlockRefs(DM, EA);
973 
974   // Finally, remove all unused phi nodes.
975   if (!(Options & BuildOptions::KeepDeadPhis))
976     removeUnusedPhis();
977 }
978 
979 // For each stack in the map DefM, push the delimiter for block B on it.
markBlock(NodeId B,DefStackMap & DefM)980 void DataFlowGraph::markBlock(NodeId B, DefStackMap &DefM) {
981   // Push block delimiters.
982   for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
983     I->second.start_block(B);
984 }
985 
986 // Remove all definitions coming from block B from each stack in DefM.
releaseBlock(NodeId B,DefStackMap & DefM)987 void DataFlowGraph::releaseBlock(NodeId B, DefStackMap &DefM) {
988   // Pop all defs from this block from the definition stack. Defs that were
989   // added to the map during the traversal of instructions will not have a
990   // delimiter, but for those, the whole stack will be emptied.
991   for (auto I = DefM.begin(), E = DefM.end(); I != E; ++I)
992     I->second.clear_block(B);
993 
994   // Finally, remove empty stacks from the map.
995   for (auto I = DefM.begin(), E = DefM.end(), NextI = I; I != E; I = NextI) {
996     NextI = std::next(I);
997     // This preserves the validity of iterators other than I.
998     if (I->second.empty())
999       DefM.erase(I);
1000   }
1001 }
1002 
1003 // Push all definitions from the instruction node IA to an appropriate
1004 // stack in DefM.
pushDefs(NodeAddr<InstrNode * > IA,DefStackMap & DefM)1005 void DataFlowGraph::pushDefs(NodeAddr<InstrNode*> IA, DefStackMap &DefM) {
1006   NodeList Defs = IA.Addr->members_if(IsDef, *this);
1007   NodeSet Visited;
1008 #ifndef NDEBUG
1009   RegisterSet Defined;
1010 #endif
1011 
1012   // The important objectives of this function are:
1013   // - to be able to handle instructions both while the graph is being
1014   //   constructed, and after the graph has been constructed, and
1015   // - maintain proper ordering of definitions on the stack for each
1016   //   register reference:
1017   //   - if there are two or more related defs in IA (i.e. coming from
1018   //     the same machine operand), then only push one def on the stack,
1019   //   - if there are multiple unrelated defs of non-overlapping
1020   //     subregisters of S, then the stack for S will have both (in an
1021   //     unspecified order), but the order does not matter from the data-
1022   //     -flow perspective.
1023 
1024   for (NodeAddr<DefNode*> DA : Defs) {
1025     if (Visited.count(DA.Id))
1026       continue;
1027     NodeList Rel = getRelatedRefs(IA, DA);
1028     NodeAddr<DefNode*> PDA = Rel.front();
1029     // Push the definition on the stack for the register and all aliases.
1030     RegisterRef RR = PDA.Addr->getRegRef();
1031 #ifndef NDEBUG
1032     // Assert if the register is defined in two or more unrelated defs.
1033     // This could happen if there are two or more def operands defining it.
1034     if (!Defined.insert(RR).second) {
1035       auto *MI = NodeAddr<StmtNode*>(IA).Addr->getCode();
1036       dbgs() << "Multiple definitions of register: "
1037              << Print<RegisterRef>(RR, *this) << " in\n  " << *MI
1038              << "in BB#" << MI->getParent()->getNumber() << '\n';
1039       llvm_unreachable(nullptr);
1040     }
1041 #endif
1042     DefM[RR].push(DA);
1043     for (auto A : RAI.getAliasSet(RR)) {
1044       assert(A != RR);
1045       DefM[A].push(DA);
1046     }
1047     // Mark all the related defs as visited.
1048     for (auto T : Rel)
1049       Visited.insert(T.Id);
1050   }
1051 }
1052 
1053 // Return the list of all reference nodes related to RA, including RA itself.
1054 // See "getNextRelated" for the meaning of a "related reference".
getRelatedRefs(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1055 NodeList DataFlowGraph::getRelatedRefs(NodeAddr<InstrNode*> IA,
1056       NodeAddr<RefNode*> RA) const {
1057   assert(IA.Id != 0 && RA.Id != 0);
1058 
1059   NodeList Refs;
1060   NodeId Start = RA.Id;
1061   do {
1062     Refs.push_back(RA);
1063     RA = getNextRelated(IA, RA);
1064   } while (RA.Id != 0 && RA.Id != Start);
1065   return Refs;
1066 }
1067 
1068 
1069 // Clear all information in the graph.
reset()1070 void DataFlowGraph::reset() {
1071   Memory.clear();
1072   Func = NodeAddr<FuncNode*>();
1073 }
1074 
1075 
1076 // Return the next reference node in the instruction node IA that is related
1077 // to RA. Conceptually, two reference nodes are related if they refer to the
1078 // same instance of a register access, but differ in flags or other minor
1079 // characteristics. Specific examples of related nodes are shadow reference
1080 // nodes.
1081 // Return the equivalent of nullptr if there are no more related references.
getNextRelated(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1082 NodeAddr<RefNode*> DataFlowGraph::getNextRelated(NodeAddr<InstrNode*> IA,
1083       NodeAddr<RefNode*> RA) const {
1084   assert(IA.Id != 0 && RA.Id != 0);
1085 
1086   auto Related = [RA](NodeAddr<RefNode*> TA) -> bool {
1087     if (TA.Addr->getKind() != RA.Addr->getKind())
1088       return false;
1089     if (TA.Addr->getRegRef() != RA.Addr->getRegRef())
1090       return false;
1091     return true;
1092   };
1093   auto RelatedStmt = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1094     return Related(TA) &&
1095            &RA.Addr->getOp() == &TA.Addr->getOp();
1096   };
1097   auto RelatedPhi = [&Related,RA](NodeAddr<RefNode*> TA) -> bool {
1098     if (!Related(TA))
1099       return false;
1100     if (TA.Addr->getKind() != NodeAttrs::Use)
1101       return true;
1102     // For phi uses, compare predecessor blocks.
1103     const NodeAddr<const PhiUseNode*> TUA = TA;
1104     const NodeAddr<const PhiUseNode*> RUA = RA;
1105     return TUA.Addr->getPredecessor() == RUA.Addr->getPredecessor();
1106   };
1107 
1108   RegisterRef RR = RA.Addr->getRegRef();
1109   if (IA.Addr->getKind() == NodeAttrs::Stmt)
1110     return RA.Addr->getNextRef(RR, RelatedStmt, true, *this);
1111   return RA.Addr->getNextRef(RR, RelatedPhi, true, *this);
1112 }
1113 
1114 // Find the next node related to RA in IA that satisfies condition P.
1115 // If such a node was found, return a pair where the second element is the
1116 // located node. If such a node does not exist, return a pair where the
1117 // first element is the element after which such a node should be inserted,
1118 // and the second element is a null-address.
1119 template <typename Predicate>
1120 std::pair<NodeAddr<RefNode*>,NodeAddr<RefNode*>>
locateNextRef(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA,Predicate P) const1121 DataFlowGraph::locateNextRef(NodeAddr<InstrNode*> IA, NodeAddr<RefNode*> RA,
1122       Predicate P) const {
1123   assert(IA.Id != 0 && RA.Id != 0);
1124 
1125   NodeAddr<RefNode*> NA;
1126   NodeId Start = RA.Id;
1127   while (true) {
1128     NA = getNextRelated(IA, RA);
1129     if (NA.Id == 0 || NA.Id == Start)
1130       break;
1131     if (P(NA))
1132       break;
1133     RA = NA;
1134   }
1135 
1136   if (NA.Id != 0 && NA.Id != Start)
1137     return std::make_pair(RA, NA);
1138   return std::make_pair(RA, NodeAddr<RefNode*>());
1139 }
1140 
1141 // Get the next shadow node in IA corresponding to RA, and optionally create
1142 // such a node if it does not exist.
getNextShadow(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA,bool Create)1143 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1144       NodeAddr<RefNode*> RA, bool Create) {
1145   assert(IA.Id != 0 && RA.Id != 0);
1146 
1147   uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1148   auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1149     return TA.Addr->getFlags() == Flags;
1150   };
1151   auto Loc = locateNextRef(IA, RA, IsShadow);
1152   if (Loc.second.Id != 0 || !Create)
1153     return Loc.second;
1154 
1155   // Create a copy of RA and mark is as shadow.
1156   NodeAddr<RefNode*> NA = cloneNode(RA);
1157   NA.Addr->setFlags(Flags | NodeAttrs::Shadow);
1158   IA.Addr->addMemberAfter(Loc.first, NA, *this);
1159   return NA;
1160 }
1161 
1162 // Get the next shadow node in IA corresponding to RA. Return null-address
1163 // if such a node does not exist.
getNextShadow(NodeAddr<InstrNode * > IA,NodeAddr<RefNode * > RA) const1164 NodeAddr<RefNode*> DataFlowGraph::getNextShadow(NodeAddr<InstrNode*> IA,
1165       NodeAddr<RefNode*> RA) const {
1166   assert(IA.Id != 0 && RA.Id != 0);
1167   uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow;
1168   auto IsShadow = [Flags] (NodeAddr<RefNode*> TA) -> bool {
1169     return TA.Addr->getFlags() == Flags;
1170   };
1171   return locateNextRef(IA, RA, IsShadow).second;
1172 }
1173 
1174 // Create a new statement node in the block node BA that corresponds to
1175 // the machine instruction MI.
buildStmt(NodeAddr<BlockNode * > BA,MachineInstr & In)1176 void DataFlowGraph::buildStmt(NodeAddr<BlockNode*> BA, MachineInstr &In) {
1177   auto SA = newStmt(BA, &In);
1178 
1179   auto isCall = [] (const MachineInstr &In) -> bool {
1180     if (In.isCall())
1181       return true;
1182     // Is tail call?
1183     if (In.isBranch())
1184       for (auto &Op : In.operands())
1185         if (Op.isGlobal() || Op.isSymbol())
1186           return true;
1187     return false;
1188   };
1189 
1190   // Collect a set of registers that this instruction implicitly uses
1191   // or defines. Implicit operands from an instruction will be ignored
1192   // unless they are listed here.
1193   RegisterSet ImpUses, ImpDefs;
1194   if (const uint16_t *ImpD = In.getDesc().getImplicitDefs())
1195     while (uint16_t R = *ImpD++)
1196       ImpDefs.insert({R, 0});
1197   if (const uint16_t *ImpU = In.getDesc().getImplicitUses())
1198     while (uint16_t R = *ImpU++)
1199       ImpUses.insert({R, 0});
1200 
1201   bool NeedsImplicit = isCall(In) || In.isInlineAsm() || In.isReturn();
1202   bool IsPredicated = TII.isPredicated(In);
1203   unsigned NumOps = In.getNumOperands();
1204 
1205   // Avoid duplicate implicit defs. This will not detect cases of implicit
1206   // defs that define registers that overlap, but it is not clear how to
1207   // interpret that in the absence of explicit defs. Overlapping explicit
1208   // defs are likely illegal already.
1209   RegisterSet DoneDefs;
1210   // Process explicit defs first.
1211   for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1212     MachineOperand &Op = In.getOperand(OpN);
1213     if (!Op.isReg() || !Op.isDef() || Op.isImplicit())
1214       continue;
1215     RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1216     uint16_t Flags = NodeAttrs::None;
1217     if (TOI.isPreserving(In, OpN))
1218       Flags |= NodeAttrs::Preserving;
1219     if (TOI.isClobbering(In, OpN))
1220       Flags |= NodeAttrs::Clobbering;
1221     if (TOI.isFixedReg(In, OpN))
1222       Flags |= NodeAttrs::Fixed;
1223     NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1224     SA.Addr->addMember(DA, *this);
1225     DoneDefs.insert(RR);
1226   }
1227 
1228   // Process implicit defs, skipping those that have already been added
1229   // as explicit.
1230   for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1231     MachineOperand &Op = In.getOperand(OpN);
1232     if (!Op.isReg() || !Op.isDef() || !Op.isImplicit())
1233       continue;
1234     RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1235     if (!NeedsImplicit && !ImpDefs.count(RR))
1236       continue;
1237     if (DoneDefs.count(RR))
1238       continue;
1239     uint16_t Flags = NodeAttrs::None;
1240     if (TOI.isPreserving(In, OpN))
1241       Flags |= NodeAttrs::Preserving;
1242     if (TOI.isClobbering(In, OpN))
1243       Flags |= NodeAttrs::Clobbering;
1244     if (TOI.isFixedReg(In, OpN))
1245       Flags |= NodeAttrs::Fixed;
1246     NodeAddr<DefNode*> DA = newDef(SA, Op, Flags);
1247     SA.Addr->addMember(DA, *this);
1248     DoneDefs.insert(RR);
1249   }
1250 
1251   for (unsigned OpN = 0; OpN < NumOps; ++OpN) {
1252     MachineOperand &Op = In.getOperand(OpN);
1253     if (!Op.isReg() || !Op.isUse())
1254       continue;
1255     RegisterRef RR = { Op.getReg(), Op.getSubReg() };
1256     // Add implicit uses on return and call instructions, and on predicated
1257     // instructions regardless of whether or not they appear in the instruction
1258     // descriptor's list.
1259     bool Implicit = Op.isImplicit();
1260     bool TakeImplicit = NeedsImplicit || IsPredicated;
1261     if (Implicit && !TakeImplicit && !ImpUses.count(RR))
1262       continue;
1263     uint16_t Flags = NodeAttrs::None;
1264     if (TOI.isFixedReg(In, OpN))
1265       Flags |= NodeAttrs::Fixed;
1266     NodeAddr<UseNode*> UA = newUse(SA, Op, Flags);
1267     SA.Addr->addMember(UA, *this);
1268   }
1269 }
1270 
1271 // Build a map that for each block will have the set of all references from
1272 // that block, and from all blocks dominated by it.
buildBlockRefs(NodeAddr<BlockNode * > BA,BlockRefsMap & RefM)1273 void DataFlowGraph::buildBlockRefs(NodeAddr<BlockNode*> BA,
1274       BlockRefsMap &RefM) {
1275   auto &Refs = RefM[BA.Id];
1276   MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode());
1277   assert(N);
1278   for (auto I : *N) {
1279     MachineBasicBlock *SB = I->getBlock();
1280     auto SBA = Func.Addr->findBlock(SB, *this);
1281     buildBlockRefs(SBA, RefM);
1282     const auto &SRs = RefM[SBA.Id];
1283     Refs.insert(SRs.begin(), SRs.end());
1284   }
1285 
1286   for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this))
1287     for (NodeAddr<RefNode*> RA : IA.Addr->members(*this))
1288       Refs.insert(RA.Addr->getRegRef());
1289 }
1290 
1291 // Scan all defs in the block node BA and record in PhiM the locations of
1292 // phi nodes corresponding to these defs.
recordDefsForDF(BlockRefsMap & PhiM,BlockRefsMap & RefM,NodeAddr<BlockNode * > BA)1293 void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, BlockRefsMap &RefM,
1294       NodeAddr<BlockNode*> BA) {
1295   // Check all defs from block BA and record them in each block in BA's
1296   // iterated dominance frontier. This information will later be used to
1297   // create phi nodes.
1298   MachineBasicBlock *BB = BA.Addr->getCode();
1299   assert(BB);
1300   auto DFLoc = MDF.find(BB);
1301   if (DFLoc == MDF.end() || DFLoc->second.empty())
1302     return;
1303 
1304   // Traverse all instructions in the block and collect the set of all
1305   // defined references. For each reference there will be a phi created
1306   // in the block's iterated dominance frontier.
1307   // This is done to make sure that each defined reference gets only one
1308   // phi node, even if it is defined multiple times.
1309   RegisterSet Defs;
1310   for (auto I : BA.Addr->members(*this)) {
1311     assert(I.Addr->getType() == NodeAttrs::Code);
1312     assert(I.Addr->getKind() == NodeAttrs::Phi ||
1313            I.Addr->getKind() == NodeAttrs::Stmt);
1314     NodeAddr<InstrNode*> IA = I;
1315     for (NodeAddr<RefNode*> RA : IA.Addr->members_if(IsDef, *this))
1316       Defs.insert(RA.Addr->getRegRef());
1317   }
1318 
1319   // Finally, add the set of defs to each block in the iterated dominance
1320   // frontier.
1321   const MachineDominanceFrontier::DomSetType &DF = DFLoc->second;
1322   SetVector<MachineBasicBlock*> IDF(DF.begin(), DF.end());
1323   for (unsigned i = 0; i < IDF.size(); ++i) {
1324     auto F = MDF.find(IDF[i]);
1325     if (F != MDF.end())
1326       IDF.insert(F->second.begin(), F->second.end());
1327   }
1328 
1329   // Get the register references that are reachable from this block.
1330   RegisterSet &Refs = RefM[BA.Id];
1331   for (auto DB : IDF) {
1332     auto DBA = Func.Addr->findBlock(DB, *this);
1333     const auto &Rs = RefM[DBA.Id];
1334     Refs.insert(Rs.begin(), Rs.end());
1335   }
1336 
1337   for (auto DB : IDF) {
1338     auto DBA = Func.Addr->findBlock(DB, *this);
1339     PhiM[DBA.Id].insert(Defs.begin(), Defs.end());
1340   }
1341 }
1342 
1343 // Given the locations of phi nodes in the map PhiM, create the phi nodes
1344 // that are located in the block node BA.
buildPhis(BlockRefsMap & PhiM,BlockRefsMap & RefM,NodeAddr<BlockNode * > BA)1345 void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, BlockRefsMap &RefM,
1346       NodeAddr<BlockNode*> BA) {
1347   // Check if this blocks has any DF defs, i.e. if there are any defs
1348   // that this block is in the iterated dominance frontier of.
1349   auto HasDF = PhiM.find(BA.Id);
1350   if (HasDF == PhiM.end() || HasDF->second.empty())
1351     return;
1352 
1353   // First, remove all R in Refs in such that there exists T in Refs
1354   // such that T covers R. In other words, only leave those refs that
1355   // are not covered by another ref (i.e. maximal with respect to covering).
1356 
1357   auto MaxCoverIn = [this] (RegisterRef RR, RegisterSet &RRs) -> RegisterRef {
1358     for (auto I : RRs)
1359       if (I != RR && RAI.covers(I, RR))
1360         RR = I;
1361     return RR;
1362   };
1363 
1364   RegisterSet MaxDF;
1365   for (auto I : HasDF->second)
1366     MaxDF.insert(MaxCoverIn(I, HasDF->second));
1367 
1368   std::vector<RegisterRef> MaxRefs;
1369   auto &RefB = RefM[BA.Id];
1370   for (auto I : MaxDF)
1371     MaxRefs.push_back(MaxCoverIn(I, RefB));
1372 
1373   // Now, for each R in MaxRefs, get the alias closure of R. If the closure
1374   // only has R in it, create a phi a def for R. Otherwise, create a phi,
1375   // and add a def for each S in the closure.
1376 
1377   // Sort the refs so that the phis will be created in a deterministic order.
1378   std::sort(MaxRefs.begin(), MaxRefs.end());
1379   // Remove duplicates.
1380   auto NewEnd = std::unique(MaxRefs.begin(), MaxRefs.end());
1381   MaxRefs.erase(NewEnd, MaxRefs.end());
1382 
1383   auto Aliased = [this,&MaxRefs](RegisterRef RR,
1384                                  std::vector<unsigned> &Closure) -> bool {
1385     for (auto I : Closure)
1386       if (RAI.alias(RR, MaxRefs[I]))
1387         return true;
1388     return false;
1389   };
1390 
1391   // Prepare a list of NodeIds of the block's predecessors.
1392   std::vector<NodeId> PredList;
1393   const MachineBasicBlock *MBB = BA.Addr->getCode();
1394   for (auto PB : MBB->predecessors()) {
1395     auto B = Func.Addr->findBlock(PB, *this);
1396     PredList.push_back(B.Id);
1397   }
1398 
1399   while (!MaxRefs.empty()) {
1400     // Put the first element in the closure, and then add all subsequent
1401     // elements from MaxRefs to it, if they alias at least one element
1402     // already in the closure.
1403     // ClosureIdx: vector of indices in MaxRefs of members of the closure.
1404     std::vector<unsigned> ClosureIdx = { 0 };
1405     for (unsigned i = 1; i != MaxRefs.size(); ++i)
1406       if (Aliased(MaxRefs[i], ClosureIdx))
1407         ClosureIdx.push_back(i);
1408 
1409     // Build a phi for the closure.
1410     unsigned CS = ClosureIdx.size();
1411     NodeAddr<PhiNode*> PA = newPhi(BA);
1412 
1413     // Add defs.
1414     for (unsigned X = 0; X != CS; ++X) {
1415       RegisterRef RR = MaxRefs[ClosureIdx[X]];
1416       uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving;
1417       NodeAddr<DefNode*> DA = newDef(PA, RR, PhiFlags);
1418       PA.Addr->addMember(DA, *this);
1419     }
1420     // Add phi uses.
1421     for (auto P : PredList) {
1422       auto PBA = addr<BlockNode*>(P);
1423       for (unsigned X = 0; X != CS; ++X) {
1424         RegisterRef RR = MaxRefs[ClosureIdx[X]];
1425         NodeAddr<PhiUseNode*> PUA = newPhiUse(PA, RR, PBA);
1426         PA.Addr->addMember(PUA, *this);
1427       }
1428     }
1429 
1430     // Erase from MaxRefs all elements in the closure.
1431     auto Begin = MaxRefs.begin();
1432     for (unsigned i = ClosureIdx.size(); i != 0; --i)
1433       MaxRefs.erase(Begin + ClosureIdx[i-1]);
1434   }
1435 }
1436 
1437 // Remove any unneeded phi nodes that were created during the build process.
removeUnusedPhis()1438 void DataFlowGraph::removeUnusedPhis() {
1439   // This will remove unused phis, i.e. phis where each def does not reach
1440   // any uses or other defs. This will not detect or remove circular phi
1441   // chains that are otherwise dead. Unused/dead phis are created during
1442   // the build process and this function is intended to remove these cases
1443   // that are easily determinable to be unnecessary.
1444 
1445   SetVector<NodeId> PhiQ;
1446   for (NodeAddr<BlockNode*> BA : Func.Addr->members(*this)) {
1447     for (auto P : BA.Addr->members_if(IsPhi, *this))
1448       PhiQ.insert(P.Id);
1449   }
1450 
1451   static auto HasUsedDef = [](NodeList &Ms) -> bool {
1452     for (auto M : Ms) {
1453       if (M.Addr->getKind() != NodeAttrs::Def)
1454         continue;
1455       NodeAddr<DefNode*> DA = M;
1456       if (DA.Addr->getReachedDef() != 0 || DA.Addr->getReachedUse() != 0)
1457         return true;
1458     }
1459     return false;
1460   };
1461 
1462   // Any phi, if it is removed, may affect other phis (make them dead).
1463   // For each removed phi, collect the potentially affected phis and add
1464   // them back to the queue.
1465   while (!PhiQ.empty()) {
1466     auto PA = addr<PhiNode*>(PhiQ[0]);
1467     PhiQ.remove(PA.Id);
1468     NodeList Refs = PA.Addr->members(*this);
1469     if (HasUsedDef(Refs))
1470       continue;
1471     for (NodeAddr<RefNode*> RA : Refs) {
1472       if (NodeId RD = RA.Addr->getReachingDef()) {
1473         auto RDA = addr<DefNode*>(RD);
1474         NodeAddr<InstrNode*> OA = RDA.Addr->getOwner(*this);
1475         if (IsPhi(OA))
1476           PhiQ.insert(OA.Id);
1477       }
1478       if (RA.Addr->isDef())
1479         unlinkDef(RA, true);
1480       else
1481         unlinkUse(RA, true);
1482     }
1483     NodeAddr<BlockNode*> BA = PA.Addr->getOwner(*this);
1484     BA.Addr->removeMember(PA, *this);
1485   }
1486 }
1487 
1488 // For a given reference node TA in an instruction node IA, connect the
1489 // reaching def of TA to the appropriate def node. Create any shadow nodes
1490 // as appropriate.
1491 template <typename T>
linkRefUp(NodeAddr<InstrNode * > IA,NodeAddr<T> TA,DefStack & DS)1492 void DataFlowGraph::linkRefUp(NodeAddr<InstrNode*> IA, NodeAddr<T> TA,
1493       DefStack &DS) {
1494   if (DS.empty())
1495     return;
1496   RegisterRef RR = TA.Addr->getRegRef();
1497   NodeAddr<T> TAP;
1498 
1499   // References from the def stack that have been examined so far.
1500   RegisterSet Defs;
1501 
1502   for (auto I = DS.top(), E = DS.bottom(); I != E; I.down()) {
1503     RegisterRef QR = I->Addr->getRegRef();
1504     auto AliasQR = [QR,this] (RegisterRef RR) -> bool {
1505       return RAI.alias(QR, RR);
1506     };
1507     bool PrecUp = RAI.covers(QR, RR);
1508     // Skip all defs that are aliased to any of the defs that we have already
1509     // seen. If we encounter a covering def, stop the stack traversal early.
1510     if (std::any_of(Defs.begin(), Defs.end(), AliasQR)) {
1511       if (PrecUp)
1512         break;
1513       continue;
1514     }
1515     // The reaching def.
1516     NodeAddr<DefNode*> RDA = *I;
1517 
1518     // Pick the reached node.
1519     if (TAP.Id == 0) {
1520       TAP = TA;
1521     } else {
1522       // Mark the existing ref as "shadow" and create a new shadow.
1523       TAP.Addr->setFlags(TAP.Addr->getFlags() | NodeAttrs::Shadow);
1524       TAP = getNextShadow(IA, TAP, true);
1525     }
1526 
1527     // Create the link.
1528     TAP.Addr->linkToDef(TAP.Id, RDA);
1529 
1530     if (PrecUp)
1531       break;
1532     Defs.insert(QR);
1533   }
1534 }
1535 
1536 // Create data-flow links for all reference nodes in the statement node SA.
linkStmtRefs(DefStackMap & DefM,NodeAddr<StmtNode * > SA)1537 void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, NodeAddr<StmtNode*> SA) {
1538   RegisterSet Defs;
1539 
1540   // Link all nodes (upwards in the data-flow) with their reaching defs.
1541   for (NodeAddr<RefNode*> RA : SA.Addr->members(*this)) {
1542     uint16_t Kind = RA.Addr->getKind();
1543     assert(Kind == NodeAttrs::Def || Kind == NodeAttrs::Use);
1544     RegisterRef RR = RA.Addr->getRegRef();
1545     // Do not process multiple defs of the same reference.
1546     if (Kind == NodeAttrs::Def && Defs.count(RR))
1547       continue;
1548     Defs.insert(RR);
1549 
1550     auto F = DefM.find(RR);
1551     if (F == DefM.end())
1552       continue;
1553     DefStack &DS = F->second;
1554     if (Kind == NodeAttrs::Use)
1555       linkRefUp<UseNode*>(SA, RA, DS);
1556     else if (Kind == NodeAttrs::Def)
1557       linkRefUp<DefNode*>(SA, RA, DS);
1558     else
1559       llvm_unreachable("Unexpected node in instruction");
1560   }
1561 }
1562 
1563 // Create data-flow links for all instructions in the block node BA. This
1564 // will include updating any phi nodes in BA.
linkBlockRefs(DefStackMap & DefM,NodeAddr<BlockNode * > BA)1565 void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, NodeAddr<BlockNode*> BA) {
1566   // Push block delimiters.
1567   markBlock(BA.Id, DefM);
1568 
1569   assert(BA.Addr && "block node address is needed to create a data-flow link");
1570   // For each non-phi instruction in the block, link all the defs and uses
1571   // to their reaching defs. For any member of the block (including phis),
1572   // push the defs on the corresponding stacks.
1573   for (NodeAddr<InstrNode*> IA : BA.Addr->members(*this)) {
1574     // Ignore phi nodes here. They will be linked part by part from the
1575     // predecessors.
1576     if (IA.Addr->getKind() == NodeAttrs::Stmt)
1577       linkStmtRefs(DefM, IA);
1578 
1579     // Push the definitions on the stack.
1580     pushDefs(IA, DefM);
1581   }
1582 
1583   // Recursively process all children in the dominator tree.
1584   MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode());
1585   for (auto I : *N) {
1586     MachineBasicBlock *SB = I->getBlock();
1587     auto SBA = Func.Addr->findBlock(SB, *this);
1588     linkBlockRefs(DefM, SBA);
1589   }
1590 
1591   // Link the phi uses from the successor blocks.
1592   auto IsUseForBA = [BA](NodeAddr<NodeBase*> NA) -> bool {
1593     if (NA.Addr->getKind() != NodeAttrs::Use)
1594       return false;
1595     assert(NA.Addr->getFlags() & NodeAttrs::PhiRef);
1596     NodeAddr<PhiUseNode*> PUA = NA;
1597     return PUA.Addr->getPredecessor() == BA.Id;
1598   };
1599   MachineBasicBlock *MBB = BA.Addr->getCode();
1600   for (auto SB : MBB->successors()) {
1601     auto SBA = Func.Addr->findBlock(SB, *this);
1602     for (NodeAddr<InstrNode*> IA : SBA.Addr->members_if(IsPhi, *this)) {
1603       // Go over each phi use associated with MBB, and link it.
1604       for (auto U : IA.Addr->members_if(IsUseForBA, *this)) {
1605         NodeAddr<PhiUseNode*> PUA = U;
1606         RegisterRef RR = PUA.Addr->getRegRef();
1607         linkRefUp<UseNode*>(IA, PUA, DefM[RR]);
1608       }
1609     }
1610   }
1611 
1612   // Pop all defs from this block from the definition stacks.
1613   releaseBlock(BA.Id, DefM);
1614 }
1615 
1616 // Remove the use node UA from any data-flow and structural links.
unlinkUseDF(NodeAddr<UseNode * > UA)1617 void DataFlowGraph::unlinkUseDF(NodeAddr<UseNode*> UA) {
1618   NodeId RD = UA.Addr->getReachingDef();
1619   NodeId Sib = UA.Addr->getSibling();
1620 
1621   if (RD == 0) {
1622     assert(Sib == 0);
1623     return;
1624   }
1625 
1626   auto RDA = addr<DefNode*>(RD);
1627   auto TA = addr<UseNode*>(RDA.Addr->getReachedUse());
1628   if (TA.Id == UA.Id) {
1629     RDA.Addr->setReachedUse(Sib);
1630     return;
1631   }
1632 
1633   while (TA.Id != 0) {
1634     NodeId S = TA.Addr->getSibling();
1635     if (S == UA.Id) {
1636       TA.Addr->setSibling(UA.Addr->getSibling());
1637       return;
1638     }
1639     TA = addr<UseNode*>(S);
1640   }
1641 }
1642 
1643 // Remove the def node DA from any data-flow and structural links.
unlinkDefDF(NodeAddr<DefNode * > DA)1644 void DataFlowGraph::unlinkDefDF(NodeAddr<DefNode*> DA) {
1645   //
1646   //         RD
1647   //         | reached
1648   //         | def
1649   //         :
1650   //         .
1651   //        +----+
1652   // ... -- | DA | -- ... -- 0  : sibling chain of DA
1653   //        +----+
1654   //         |  | reached
1655   //         |  : def
1656   //         |  .
1657   //         | ...  : Siblings (defs)
1658   //         |
1659   //         : reached
1660   //         . use
1661   //        ... : sibling chain of reached uses
1662 
1663   NodeId RD = DA.Addr->getReachingDef();
1664 
1665   // Visit all siblings of the reached def and reset their reaching defs.
1666   // Also, defs reached by DA are now "promoted" to being reached by RD,
1667   // so all of them will need to be spliced into the sibling chain where
1668   // DA belongs.
1669   auto getAllNodes = [this] (NodeId N) -> NodeList {
1670     NodeList Res;
1671     while (N) {
1672       auto RA = addr<RefNode*>(N);
1673       // Keep the nodes in the exact sibling order.
1674       Res.push_back(RA);
1675       N = RA.Addr->getSibling();
1676     }
1677     return Res;
1678   };
1679   NodeList ReachedDefs = getAllNodes(DA.Addr->getReachedDef());
1680   NodeList ReachedUses = getAllNodes(DA.Addr->getReachedUse());
1681 
1682   if (RD == 0) {
1683     for (NodeAddr<RefNode*> I : ReachedDefs)
1684       I.Addr->setSibling(0);
1685     for (NodeAddr<RefNode*> I : ReachedUses)
1686       I.Addr->setSibling(0);
1687   }
1688   for (NodeAddr<DefNode*> I : ReachedDefs)
1689     I.Addr->setReachingDef(RD);
1690   for (NodeAddr<UseNode*> I : ReachedUses)
1691     I.Addr->setReachingDef(RD);
1692 
1693   NodeId Sib = DA.Addr->getSibling();
1694   if (RD == 0) {
1695     assert(Sib == 0);
1696     return;
1697   }
1698 
1699   // Update the reaching def node and remove DA from the sibling list.
1700   auto RDA = addr<DefNode*>(RD);
1701   auto TA = addr<DefNode*>(RDA.Addr->getReachedDef());
1702   if (TA.Id == DA.Id) {
1703     // If DA is the first reached def, just update the RD's reached def
1704     // to the DA's sibling.
1705     RDA.Addr->setReachedDef(Sib);
1706   } else {
1707     // Otherwise, traverse the sibling list of the reached defs and remove
1708     // DA from it.
1709     while (TA.Id != 0) {
1710       NodeId S = TA.Addr->getSibling();
1711       if (S == DA.Id) {
1712         TA.Addr->setSibling(Sib);
1713         break;
1714       }
1715       TA = addr<DefNode*>(S);
1716     }
1717   }
1718 
1719   // Splice the DA's reached defs into the RDA's reached def chain.
1720   if (!ReachedDefs.empty()) {
1721     auto Last = NodeAddr<DefNode*>(ReachedDefs.back());
1722     Last.Addr->setSibling(RDA.Addr->getReachedDef());
1723     RDA.Addr->setReachedDef(ReachedDefs.front().Id);
1724   }
1725   // Splice the DA's reached uses into the RDA's reached use chain.
1726   if (!ReachedUses.empty()) {
1727     auto Last = NodeAddr<UseNode*>(ReachedUses.back());
1728     Last.Addr->setSibling(RDA.Addr->getReachedUse());
1729     RDA.Addr->setReachedUse(ReachedUses.front().Id);
1730   }
1731 }
1732