• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===- GuardWidening.cpp - ---- Guard widening ----------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the guard widening pass.  The semantics of the
11 // @llvm.experimental.guard intrinsic lets LLVM transform it so that it fails
12 // more often that it did before the transform.  This optimization is called
13 // "widening" and can be used hoist and common runtime checks in situations like
14 // these:
15 //
16 //    %cmp0 = 7 u< Length
17 //    call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
18 //    call @unknown_side_effects()
19 //    %cmp1 = 9 u< Length
20 //    call @llvm.experimental.guard(i1 %cmp1) [ "deopt"(...) ]
21 //    ...
22 //
23 // =>
24 //
25 //    %cmp0 = 9 u< Length
26 //    call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
27 //    call @unknown_side_effects()
28 //    ...
29 //
30 // If %cmp0 is false, @llvm.experimental.guard will "deoptimize" back to a
31 // generic implementation of the same function, which will have the correct
32 // semantics from that point onward.  It is always _legal_ to deoptimize (so
33 // replacing %cmp0 with false is "correct"), though it may not always be
34 // profitable to do so.
35 //
36 // NB! This pass is a work in progress.  It hasn't been tuned to be "production
37 // ready" yet.  It is known to have quadriatic running time and will not scale
38 // to large numbers of guards
39 //
40 //===----------------------------------------------------------------------===//
41 
42 #include "llvm/Transforms/Scalar/GuardWidening.h"
43 #include "llvm/Pass.h"
44 #include "llvm/ADT/DenseMap.h"
45 #include "llvm/ADT/DepthFirstIterator.h"
46 #include "llvm/Analysis/LoopInfo.h"
47 #include "llvm/Analysis/PostDominators.h"
48 #include "llvm/Analysis/ValueTracking.h"
49 #include "llvm/IR/Dominators.h"
50 #include "llvm/IR/IntrinsicInst.h"
51 #include "llvm/IR/PatternMatch.h"
52 #include "llvm/Support/Debug.h"
53 #include "llvm/Transforms/Scalar.h"
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "guard-widening"
58 
59 namespace {
60 
61 class GuardWideningImpl {
62   DominatorTree &DT;
63   PostDominatorTree &PDT;
64   LoopInfo &LI;
65 
66   /// The set of guards whose conditions have been widened into dominating
67   /// guards.
68   SmallVector<IntrinsicInst *, 16> EliminatedGuards;
69 
70   /// The set of guards which have been widened to include conditions to other
71   /// guards.
72   DenseSet<IntrinsicInst *> WidenedGuards;
73 
74   /// Try to eliminate guard \p Guard by widening it into an earlier dominating
75   /// guard.  \p DFSI is the DFS iterator on the dominator tree that is
76   /// currently visiting the block containing \p Guard, and \p GuardsPerBlock
77   /// maps BasicBlocks to the set of guards seen in that block.
78   bool eliminateGuardViaWidening(
79       IntrinsicInst *Guard, const df_iterator<DomTreeNode *> &DFSI,
80       const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> &
81           GuardsPerBlock);
82 
83   /// Used to keep track of which widening potential is more effective.
84   enum WideningScore {
85     /// Don't widen.
86     WS_IllegalOrNegative,
87 
88     /// Widening is performance neutral as far as the cycles spent in check
89     /// conditions goes (but can still help, e.g., code layout, having less
90     /// deopt state).
91     WS_Neutral,
92 
93     /// Widening is profitable.
94     WS_Positive,
95 
96     /// Widening is very profitable.  Not significantly different from \c
97     /// WS_Positive, except by the order.
98     WS_VeryPositive
99   };
100 
101   static StringRef scoreTypeToString(WideningScore WS);
102 
103   /// Compute the score for widening the condition in \p DominatedGuard
104   /// (contained in \p DominatedGuardLoop) into \p DominatingGuard (contained in
105   /// \p DominatingGuardLoop).
106   WideningScore computeWideningScore(IntrinsicInst *DominatedGuard,
107                                      Loop *DominatedGuardLoop,
108                                      IntrinsicInst *DominatingGuard,
109                                      Loop *DominatingGuardLoop);
110 
111   /// Helper to check if \p V can be hoisted to \p InsertPos.
isAvailableAt(Value * V,Instruction * InsertPos)112   bool isAvailableAt(Value *V, Instruction *InsertPos) {
113     SmallPtrSet<Instruction *, 8> Visited;
114     return isAvailableAt(V, InsertPos, Visited);
115   }
116 
117   bool isAvailableAt(Value *V, Instruction *InsertPos,
118                      SmallPtrSetImpl<Instruction *> &Visited);
119 
120   /// Helper to hoist \p V to \p InsertPos.  Guaranteed to succeed if \c
121   /// isAvailableAt returned true.
122   void makeAvailableAt(Value *V, Instruction *InsertPos);
123 
124   /// Common helper used by \c widenGuard and \c isWideningCondProfitable.  Try
125   /// to generate an expression computing the logical AND of \p Cond0 and \p
126   /// Cond1.  Return true if the expression computing the AND is only as
127   /// expensive as computing one of the two. If \p InsertPt is true then
128   /// actually generate the resulting expression, make it available at \p
129   /// InsertPt and return it in \p Result (else no change to the IR is made).
130   bool widenCondCommon(Value *Cond0, Value *Cond1, Instruction *InsertPt,
131                        Value *&Result);
132 
133   /// Represents a range check of the form \c Base + \c Offset u< \c Length,
134   /// with the constraint that \c Length is not negative.  \c CheckInst is the
135   /// pre-existing instruction in the IR that computes the result of this range
136   /// check.
137   class RangeCheck {
138     Value *Base;
139     ConstantInt *Offset;
140     Value *Length;
141     ICmpInst *CheckInst;
142 
143   public:
RangeCheck(Value * Base,ConstantInt * Offset,Value * Length,ICmpInst * CheckInst)144     explicit RangeCheck(Value *Base, ConstantInt *Offset, Value *Length,
145                         ICmpInst *CheckInst)
146         : Base(Base), Offset(Offset), Length(Length), CheckInst(CheckInst) {}
147 
setBase(Value * NewBase)148     void setBase(Value *NewBase) { Base = NewBase; }
setOffset(ConstantInt * NewOffset)149     void setOffset(ConstantInt *NewOffset) { Offset = NewOffset; }
150 
getBase() const151     Value *getBase() const { return Base; }
getOffset() const152     ConstantInt *getOffset() const { return Offset; }
getOffsetValue() const153     const APInt &getOffsetValue() const { return getOffset()->getValue(); }
getLength() const154     Value *getLength() const { return Length; };
getCheckInst() const155     ICmpInst *getCheckInst() const { return CheckInst; }
156 
print(raw_ostream & OS,bool PrintTypes=false)157     void print(raw_ostream &OS, bool PrintTypes = false) {
158       OS << "Base: ";
159       Base->printAsOperand(OS, PrintTypes);
160       OS << " Offset: ";
161       Offset->printAsOperand(OS, PrintTypes);
162       OS << " Length: ";
163       Length->printAsOperand(OS, PrintTypes);
164     }
165 
dump()166     LLVM_DUMP_METHOD void dump() {
167       print(dbgs());
168       dbgs() << "\n";
169     }
170   };
171 
172   /// Parse \p CheckCond into a conjunction (logical-and) of range checks; and
173   /// append them to \p Checks.  Returns true on success, may clobber \c Checks
174   /// on failure.
parseRangeChecks(Value * CheckCond,SmallVectorImpl<RangeCheck> & Checks)175   bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks) {
176     SmallPtrSet<Value *, 8> Visited;
177     return parseRangeChecks(CheckCond, Checks, Visited);
178   }
179 
180   bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks,
181                         SmallPtrSetImpl<Value *> &Visited);
182 
183   /// Combine the checks in \p Checks into a smaller set of checks and append
184   /// them into \p CombinedChecks.  Return true on success (i.e. all of checks
185   /// in \p Checks were combined into \p CombinedChecks).  Clobbers \p Checks
186   /// and \p CombinedChecks on success and on failure.
187   bool combineRangeChecks(SmallVectorImpl<RangeCheck> &Checks,
188                           SmallVectorImpl<RangeCheck> &CombinedChecks);
189 
190   /// Can we compute the logical AND of \p Cond0 and \p Cond1 for the price of
191   /// computing only one of the two expressions?
isWideningCondProfitable(Value * Cond0,Value * Cond1)192   bool isWideningCondProfitable(Value *Cond0, Value *Cond1) {
193     Value *ResultUnused;
194     return widenCondCommon(Cond0, Cond1, /*InsertPt=*/nullptr, ResultUnused);
195   }
196 
197   /// Widen \p ToWiden to fail if \p NewCondition is false (in addition to
198   /// whatever it is already checking).
widenGuard(IntrinsicInst * ToWiden,Value * NewCondition)199   void widenGuard(IntrinsicInst *ToWiden, Value *NewCondition) {
200     Value *Result;
201     widenCondCommon(ToWiden->getArgOperand(0), NewCondition, ToWiden, Result);
202     ToWiden->setArgOperand(0, Result);
203   }
204 
205 public:
GuardWideningImpl(DominatorTree & DT,PostDominatorTree & PDT,LoopInfo & LI)206   explicit GuardWideningImpl(DominatorTree &DT, PostDominatorTree &PDT,
207                              LoopInfo &LI)
208       : DT(DT), PDT(PDT), LI(LI) {}
209 
210   /// The entry point for this pass.
211   bool run();
212 };
213 
214 struct GuardWideningLegacyPass : public FunctionPass {
215   static char ID;
216   GuardWideningPass Impl;
217 
GuardWideningLegacyPass__anonbd0986bc0111::GuardWideningLegacyPass218   GuardWideningLegacyPass() : FunctionPass(ID) {
219     initializeGuardWideningLegacyPassPass(*PassRegistry::getPassRegistry());
220   }
221 
runOnFunction__anonbd0986bc0111::GuardWideningLegacyPass222   bool runOnFunction(Function &F) override {
223     if (skipFunction(F))
224       return false;
225     return GuardWideningImpl(
226                getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
227                getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree(),
228                getAnalysis<LoopInfoWrapperPass>().getLoopInfo()).run();
229   }
230 
getAnalysisUsage__anonbd0986bc0111::GuardWideningLegacyPass231   void getAnalysisUsage(AnalysisUsage &AU) const override {
232     AU.setPreservesCFG();
233     AU.addRequired<DominatorTreeWrapperPass>();
234     AU.addRequired<PostDominatorTreeWrapperPass>();
235     AU.addRequired<LoopInfoWrapperPass>();
236   }
237 };
238 
239 }
240 
run()241 bool GuardWideningImpl::run() {
242   using namespace llvm::PatternMatch;
243 
244   DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> GuardsInBlock;
245   bool Changed = false;
246 
247   for (auto DFI = df_begin(DT.getRootNode()), DFE = df_end(DT.getRootNode());
248        DFI != DFE; ++DFI) {
249     auto *BB = (*DFI)->getBlock();
250     auto &CurrentList = GuardsInBlock[BB];
251 
252     for (auto &I : *BB)
253       if (match(&I, m_Intrinsic<Intrinsic::experimental_guard>()))
254         CurrentList.push_back(cast<IntrinsicInst>(&I));
255 
256     for (auto *II : CurrentList)
257       Changed |= eliminateGuardViaWidening(II, DFI, GuardsInBlock);
258   }
259 
260   for (auto *II : EliminatedGuards)
261     if (!WidenedGuards.count(II))
262       II->eraseFromParent();
263 
264   return Changed;
265 }
266 
eliminateGuardViaWidening(IntrinsicInst * GuardInst,const df_iterator<DomTreeNode * > & DFSI,const DenseMap<BasicBlock *,SmallVector<IntrinsicInst *,8>> & GuardsInBlock)267 bool GuardWideningImpl::eliminateGuardViaWidening(
268     IntrinsicInst *GuardInst, const df_iterator<DomTreeNode *> &DFSI,
269     const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> &
270         GuardsInBlock) {
271   IntrinsicInst *BestSoFar = nullptr;
272   auto BestScoreSoFar = WS_IllegalOrNegative;
273   auto *GuardInstLoop = LI.getLoopFor(GuardInst->getParent());
274 
275   // In the set of dominating guards, find the one we can merge GuardInst with
276   // for the most profit.
277   for (unsigned i = 0, e = DFSI.getPathLength(); i != e; ++i) {
278     auto *CurBB = DFSI.getPath(i)->getBlock();
279     auto *CurLoop = LI.getLoopFor(CurBB);
280     assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!");
281     const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
282 
283     auto I = GuardsInCurBB.begin();
284     auto E = GuardsInCurBB.end();
285 
286 #ifndef NDEBUG
287     {
288       unsigned Index = 0;
289       for (auto &I : *CurBB) {
290         if (Index == GuardsInCurBB.size())
291           break;
292         if (GuardsInCurBB[Index] == &I)
293           Index++;
294       }
295       assert(Index == GuardsInCurBB.size() &&
296              "Guards expected to be in order!");
297     }
298 #endif
299 
300     assert((i == (e - 1)) == (GuardInst->getParent() == CurBB) && "Bad DFS?");
301 
302     if (i == (e - 1)) {
303       // Corner case: make sure we're only looking at guards strictly dominating
304       // GuardInst when visiting GuardInst->getParent().
305       auto NewEnd = std::find(I, E, GuardInst);
306       assert(NewEnd != E && "GuardInst not in its own block?");
307       E = NewEnd;
308     }
309 
310     for (auto *Candidate : make_range(I, E)) {
311       auto Score =
312           computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop);
313       DEBUG(dbgs() << "Score between " << *GuardInst->getArgOperand(0)
314                    << " and " << *Candidate->getArgOperand(0) << " is "
315                    << scoreTypeToString(Score) << "\n");
316       if (Score > BestScoreSoFar) {
317         BestScoreSoFar = Score;
318         BestSoFar = Candidate;
319       }
320     }
321   }
322 
323   if (BestScoreSoFar == WS_IllegalOrNegative) {
324     DEBUG(dbgs() << "Did not eliminate guard " << *GuardInst << "\n");
325     return false;
326   }
327 
328   assert(BestSoFar != GuardInst && "Should have never visited same guard!");
329   assert(DT.dominates(BestSoFar, GuardInst) && "Should be!");
330 
331   DEBUG(dbgs() << "Widening " << *GuardInst << " into " << *BestSoFar
332                << " with score " << scoreTypeToString(BestScoreSoFar) << "\n");
333   widenGuard(BestSoFar, GuardInst->getArgOperand(0));
334   GuardInst->setArgOperand(0, ConstantInt::getTrue(GuardInst->getContext()));
335   EliminatedGuards.push_back(GuardInst);
336   WidenedGuards.insert(BestSoFar);
337   return true;
338 }
339 
computeWideningScore(IntrinsicInst * DominatedGuard,Loop * DominatedGuardLoop,IntrinsicInst * DominatingGuard,Loop * DominatingGuardLoop)340 GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
341     IntrinsicInst *DominatedGuard, Loop *DominatedGuardLoop,
342     IntrinsicInst *DominatingGuard, Loop *DominatingGuardLoop) {
343   bool HoistingOutOfLoop = false;
344 
345   if (DominatingGuardLoop != DominatedGuardLoop) {
346     if (DominatingGuardLoop &&
347         !DominatingGuardLoop->contains(DominatedGuardLoop))
348       return WS_IllegalOrNegative;
349 
350     HoistingOutOfLoop = true;
351   }
352 
353   if (!isAvailableAt(DominatedGuard->getArgOperand(0), DominatingGuard))
354     return WS_IllegalOrNegative;
355 
356   bool HoistingOutOfIf =
357       !PDT.dominates(DominatedGuard->getParent(), DominatingGuard->getParent());
358 
359   if (isWideningCondProfitable(DominatedGuard->getArgOperand(0),
360                                DominatingGuard->getArgOperand(0)))
361     return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive;
362 
363   if (HoistingOutOfLoop)
364     return WS_Positive;
365 
366   return HoistingOutOfIf ? WS_IllegalOrNegative : WS_Neutral;
367 }
368 
isAvailableAt(Value * V,Instruction * Loc,SmallPtrSetImpl<Instruction * > & Visited)369 bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc,
370                                       SmallPtrSetImpl<Instruction *> &Visited) {
371   auto *Inst = dyn_cast<Instruction>(V);
372   if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst))
373     return true;
374 
375   if (!isSafeToSpeculativelyExecute(Inst, Loc, &DT) ||
376       Inst->mayReadFromMemory())
377     return false;
378 
379   Visited.insert(Inst);
380 
381   // We only want to go _up_ the dominance chain when recursing.
382   assert(!isa<PHINode>(Loc) &&
383          "PHIs should return false for isSafeToSpeculativelyExecute");
384   assert(DT.isReachableFromEntry(Inst->getParent()) &&
385          "We did a DFS from the block entry!");
386   return all_of(Inst->operands(),
387                 [&](Value *Op) { return isAvailableAt(Op, Loc, Visited); });
388 }
389 
makeAvailableAt(Value * V,Instruction * Loc)390 void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) {
391   auto *Inst = dyn_cast<Instruction>(V);
392   if (!Inst || DT.dominates(Inst, Loc))
393     return;
394 
395   assert(isSafeToSpeculativelyExecute(Inst, Loc, &DT) &&
396          !Inst->mayReadFromMemory() && "Should've checked with isAvailableAt!");
397 
398   for (Value *Op : Inst->operands())
399     makeAvailableAt(Op, Loc);
400 
401   Inst->moveBefore(Loc);
402 }
403 
widenCondCommon(Value * Cond0,Value * Cond1,Instruction * InsertPt,Value * & Result)404 bool GuardWideningImpl::widenCondCommon(Value *Cond0, Value *Cond1,
405                                         Instruction *InsertPt, Value *&Result) {
406   using namespace llvm::PatternMatch;
407 
408   {
409     // L >u C0 && L >u C1  ->  L >u max(C0, C1)
410     ConstantInt *RHS0, *RHS1;
411     Value *LHS;
412     ICmpInst::Predicate Pred0, Pred1;
413     if (match(Cond0, m_ICmp(Pred0, m_Value(LHS), m_ConstantInt(RHS0))) &&
414         match(Cond1, m_ICmp(Pred1, m_Specific(LHS), m_ConstantInt(RHS1)))) {
415 
416       ConstantRange CR0 =
417           ConstantRange::makeExactICmpRegion(Pred0, RHS0->getValue());
418       ConstantRange CR1 =
419           ConstantRange::makeExactICmpRegion(Pred1, RHS1->getValue());
420 
421       // SubsetIntersect is a subset of the actual mathematical intersection of
422       // CR0 and CR1, while SupersetIntersect is a superset of the actual
423       // mathematical intersection.  If these two ConstantRanges are equal, then
424       // we know we were able to represent the actual mathematical intersection
425       // of CR0 and CR1, and can use the same to generate an icmp instruction.
426       //
427       // Given what we're doing here and the semantics of guards, it would
428       // actually be correct to just use SubsetIntersect, but that may be too
429       // aggressive in cases we care about.
430       auto SubsetIntersect = CR0.inverse().unionWith(CR1.inverse()).inverse();
431       auto SupersetIntersect = CR0.intersectWith(CR1);
432 
433       APInt NewRHSAP;
434       CmpInst::Predicate Pred;
435       if (SubsetIntersect == SupersetIntersect &&
436           SubsetIntersect.getEquivalentICmp(Pred, NewRHSAP)) {
437         if (InsertPt) {
438           ConstantInt *NewRHS = ConstantInt::get(Cond0->getContext(), NewRHSAP);
439           Result = new ICmpInst(InsertPt, Pred, LHS, NewRHS, "wide.chk");
440         }
441         return true;
442       }
443     }
444   }
445 
446   {
447     SmallVector<GuardWideningImpl::RangeCheck, 4> Checks, CombinedChecks;
448     if (parseRangeChecks(Cond0, Checks) && parseRangeChecks(Cond1, Checks) &&
449         combineRangeChecks(Checks, CombinedChecks)) {
450       if (InsertPt) {
451         Result = nullptr;
452         for (auto &RC : CombinedChecks) {
453           makeAvailableAt(RC.getCheckInst(), InsertPt);
454           if (Result)
455             Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result, "",
456                                                InsertPt);
457           else
458             Result = RC.getCheckInst();
459         }
460 
461         Result->setName("wide.chk");
462       }
463       return true;
464     }
465   }
466 
467   // Base case -- just logical-and the two conditions together.
468 
469   if (InsertPt) {
470     makeAvailableAt(Cond0, InsertPt);
471     makeAvailableAt(Cond1, InsertPt);
472 
473     Result = BinaryOperator::CreateAnd(Cond0, Cond1, "wide.chk", InsertPt);
474   }
475 
476   // We were not able to compute Cond0 AND Cond1 for the price of one.
477   return false;
478 }
479 
parseRangeChecks(Value * CheckCond,SmallVectorImpl<GuardWideningImpl::RangeCheck> & Checks,SmallPtrSetImpl<Value * > & Visited)480 bool GuardWideningImpl::parseRangeChecks(
481     Value *CheckCond, SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
482     SmallPtrSetImpl<Value *> &Visited) {
483   if (!Visited.insert(CheckCond).second)
484     return true;
485 
486   using namespace llvm::PatternMatch;
487 
488   {
489     Value *AndLHS, *AndRHS;
490     if (match(CheckCond, m_And(m_Value(AndLHS), m_Value(AndRHS))))
491       return parseRangeChecks(AndLHS, Checks) &&
492              parseRangeChecks(AndRHS, Checks);
493   }
494 
495   auto *IC = dyn_cast<ICmpInst>(CheckCond);
496   if (!IC || !IC->getOperand(0)->getType()->isIntegerTy() ||
497       (IC->getPredicate() != ICmpInst::ICMP_ULT &&
498        IC->getPredicate() != ICmpInst::ICMP_UGT))
499     return false;
500 
501   Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
502   if (IC->getPredicate() == ICmpInst::ICMP_UGT)
503     std::swap(CmpLHS, CmpRHS);
504 
505   auto &DL = IC->getModule()->getDataLayout();
506 
507   GuardWideningImpl::RangeCheck Check(
508       CmpLHS, cast<ConstantInt>(ConstantInt::getNullValue(CmpRHS->getType())),
509       CmpRHS, IC);
510 
511   if (!isKnownNonNegative(Check.getLength(), DL))
512     return false;
513 
514   // What we have in \c Check now is a correct interpretation of \p CheckCond.
515   // Try to see if we can move some constant offsets into the \c Offset field.
516 
517   bool Changed;
518   auto &Ctx = CheckCond->getContext();
519 
520   do {
521     Value *OpLHS;
522     ConstantInt *OpRHS;
523     Changed = false;
524 
525 #ifndef NDEBUG
526     auto *BaseInst = dyn_cast<Instruction>(Check.getBase());
527     assert((!BaseInst || DT.isReachableFromEntry(BaseInst->getParent())) &&
528            "Unreachable instruction?");
529 #endif
530 
531     if (match(Check.getBase(), m_Add(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
532       Check.setBase(OpLHS);
533       APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
534       Check.setOffset(ConstantInt::get(Ctx, NewOffset));
535       Changed = true;
536     } else if (match(Check.getBase(),
537                      m_Or(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
538       unsigned BitWidth = OpLHS->getType()->getScalarSizeInBits();
539       APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
540       computeKnownBits(OpLHS, KnownZero, KnownOne, DL);
541       if ((OpRHS->getValue() & KnownZero) == OpRHS->getValue()) {
542         Check.setBase(OpLHS);
543         APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
544         Check.setOffset(ConstantInt::get(Ctx, NewOffset));
545         Changed = true;
546       }
547     }
548   } while (Changed);
549 
550   Checks.push_back(Check);
551   return true;
552 }
553 
combineRangeChecks(SmallVectorImpl<GuardWideningImpl::RangeCheck> & Checks,SmallVectorImpl<GuardWideningImpl::RangeCheck> & RangeChecksOut)554 bool GuardWideningImpl::combineRangeChecks(
555     SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
556     SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) {
557   unsigned OldCount = Checks.size();
558   while (!Checks.empty()) {
559     // Pick all of the range checks with a specific base and length, and try to
560     // merge them.
561     Value *CurrentBase = Checks.front().getBase();
562     Value *CurrentLength = Checks.front().getLength();
563 
564     SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks;
565 
566     auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) {
567       return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength;
568     };
569 
570     std::copy_if(Checks.begin(), Checks.end(),
571                  std::back_inserter(CurrentChecks), IsCurrentCheck);
572     Checks.erase(remove_if(Checks, IsCurrentCheck), Checks.end());
573 
574     assert(CurrentChecks.size() != 0 && "We know we have at least one!");
575 
576     if (CurrentChecks.size() < 3) {
577       RangeChecksOut.insert(RangeChecksOut.end(), CurrentChecks.begin(),
578                             CurrentChecks.end());
579       continue;
580     }
581 
582     // CurrentChecks.size() will typically be 3 here, but so far there has been
583     // no need to hard-code that fact.
584 
585     std::sort(CurrentChecks.begin(), CurrentChecks.end(),
586               [&](const GuardWideningImpl::RangeCheck &LHS,
587                   const GuardWideningImpl::RangeCheck &RHS) {
588       return LHS.getOffsetValue().slt(RHS.getOffsetValue());
589     });
590 
591     // Note: std::sort should not invalidate the ChecksStart iterator.
592 
593     ConstantInt *MinOffset = CurrentChecks.front().getOffset(),
594                 *MaxOffset = CurrentChecks.back().getOffset();
595 
596     unsigned BitWidth = MaxOffset->getValue().getBitWidth();
597     if ((MaxOffset->getValue() - MinOffset->getValue())
598             .ugt(APInt::getSignedMinValue(BitWidth)))
599       return false;
600 
601     APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue();
602     const APInt &HighOffset = MaxOffset->getValue();
603     auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) {
604       return (HighOffset - RC.getOffsetValue()).ult(MaxDiff);
605     };
606 
607     if (MaxDiff.isMinValue() ||
608         !std::all_of(std::next(CurrentChecks.begin()), CurrentChecks.end(),
609                      OffsetOK))
610       return false;
611 
612     // We have a series of f+1 checks as:
613     //
614     //   I+k_0 u< L   ... Chk_0
615     //   I_k_1 u< L   ... Chk_1
616     //   ...
617     //   I_k_f u< L   ... Chk_(f+1)
618     //
619     //     with forall i in [0,f): k_f-k_i u< k_f-k_0  ... Precond_0
620     //          k_f-k_0 u< INT_MIN+k_f                 ... Precond_1
621     //          k_f != k_0                             ... Precond_2
622     //
623     // Claim:
624     //   Chk_0 AND Chk_(f+1)  implies all the other checks
625     //
626     // Informal proof sketch:
627     //
628     // We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap
629     // (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and
630     // thus I+k_f is the greatest unsigned value in that range.
631     //
632     // This combined with Ckh_(f+1) shows that everything in that range is u< L.
633     // Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1)
634     // lie in [I+k_0,I+k_f], this proving our claim.
635     //
636     // To see that [I+k_0,I+k_f] is not a wrapping range, note that there are
637     // two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal
638     // since k_0 != k_f).  In the former case, [I+k_0,I+k_f] is not a wrapping
639     // range by definition, and the latter case is impossible:
640     //
641     //   0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1)
642     //   xxxxxx             xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
643     //
644     // For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted
645     // with 'x' above) to be at least >u INT_MIN.
646 
647     RangeChecksOut.emplace_back(CurrentChecks.front());
648     RangeChecksOut.emplace_back(CurrentChecks.back());
649   }
650 
651   assert(RangeChecksOut.size() <= OldCount && "We pessimized!");
652   return RangeChecksOut.size() != OldCount;
653 }
654 
run(Function & F,AnalysisManager<Function> & AM)655 PreservedAnalyses GuardWideningPass::run(Function &F,
656                                          AnalysisManager<Function> &AM) {
657   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
658   auto &LI = AM.getResult<LoopAnalysis>(F);
659   auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
660   bool Changed = GuardWideningImpl(DT, PDT, LI).run();
661   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
662 }
663 
scoreTypeToString(WideningScore WS)664 StringRef GuardWideningImpl::scoreTypeToString(WideningScore WS) {
665   switch (WS) {
666   case WS_IllegalOrNegative:
667     return "IllegalOrNegative";
668   case WS_Neutral:
669     return "Neutral";
670   case WS_Positive:
671     return "Positive";
672   case WS_VeryPositive:
673     return "VeryPositive";
674   }
675 
676   llvm_unreachable("Fully covered switch above!");
677 }
678 
679 char GuardWideningLegacyPass::ID = 0;
680 
681 INITIALIZE_PASS_BEGIN(GuardWideningLegacyPass, "guard-widening", "Widen guards",
682                       false, false)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)683 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
684 INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
685 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
686 INITIALIZE_PASS_END(GuardWideningLegacyPass, "guard-widening", "Widen guards",
687                     false, false)
688 
689 FunctionPass *llvm::createGuardWideningPass() {
690   return new GuardWideningLegacyPass();
691 }
692