1 //===- AMDGPUInline.cpp - Code to perform simple function inlining --------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This is AMDGPU specific replacement of the standard inliner.
12 /// The main purpose is to account for the fact that calls not only expensive
13 /// on the AMDGPU, but much more expensive if a private memory pointer is
14 /// passed to a function as an argument. In this situation, we are unable to
15 /// eliminate private memory in the caller unless inlined and end up with slow
16 /// and expensive scratch access. Thus, we boost the inline threshold for such
17 /// functions here.
18 ///
19 //===----------------------------------------------------------------------===//
20
21
22 #include "AMDGPU.h"
23 #include "llvm/Transforms/IPO.h"
24 #include "llvm/Analysis/AssumptionCache.h"
25 #include "llvm/Analysis/CallGraph.h"
26 #include "llvm/Analysis/InlineCost.h"
27 #include "llvm/Analysis/ValueTracking.h"
28 #include "llvm/Analysis/TargetTransformInfo.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/Instructions.h"
32 #include "llvm/IR/Module.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/Support/CommandLine.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Transforms/IPO/Inliner.h"
37
38 using namespace llvm;
39
40 #define DEBUG_TYPE "inline"
41
42 static cl::opt<int>
43 ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(2200),
44 cl::desc("Cost of alloca argument"));
45
46 // If the amount of scratch memory to eliminate exceeds our ability to allocate
47 // it into registers we gain nothing by agressively inlining functions for that
48 // heuristic.
49 static cl::opt<unsigned>
50 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden, cl::init(256),
51 cl::desc("Maximum alloca size to use for inline cost"));
52
53 namespace {
54
55 class AMDGPUInliner : public LegacyInlinerBase {
56
57 public:
AMDGPUInliner()58 AMDGPUInliner() : LegacyInlinerBase(ID) {
59 initializeAMDGPUInlinerPass(*PassRegistry::getPassRegistry());
60 Params = getInlineParams();
61 }
62
63 static char ID; // Pass identification, replacement for typeid
64
65 unsigned getInlineThreshold(CallSite CS) const;
66
67 InlineCost getInlineCost(CallSite CS) override;
68
69 bool runOnSCC(CallGraphSCC &SCC) override;
70
71 void getAnalysisUsage(AnalysisUsage &AU) const override;
72
73 private:
74 TargetTransformInfoWrapperPass *TTIWP;
75
76 InlineParams Params;
77 };
78
79 } // end anonymous namespace
80
81 char AMDGPUInliner::ID = 0;
82 INITIALIZE_PASS_BEGIN(AMDGPUInliner, "amdgpu-inline",
83 "AMDGPU Function Integration/Inlining", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)84 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
85 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
86 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
87 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
88 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
89 INITIALIZE_PASS_END(AMDGPUInliner, "amdgpu-inline",
90 "AMDGPU Function Integration/Inlining", false, false)
91
92 Pass *llvm::createAMDGPUFunctionInliningPass() { return new AMDGPUInliner(); }
93
runOnSCC(CallGraphSCC & SCC)94 bool AMDGPUInliner::runOnSCC(CallGraphSCC &SCC) {
95 TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>();
96 return LegacyInlinerBase::runOnSCC(SCC);
97 }
98
getAnalysisUsage(AnalysisUsage & AU) const99 void AMDGPUInliner::getAnalysisUsage(AnalysisUsage &AU) const {
100 AU.addRequired<TargetTransformInfoWrapperPass>();
101 LegacyInlinerBase::getAnalysisUsage(AU);
102 }
103
getInlineThreshold(CallSite CS) const104 unsigned AMDGPUInliner::getInlineThreshold(CallSite CS) const {
105 int Thres = Params.DefaultThreshold;
106
107 Function *Caller = CS.getCaller();
108 // Listen to the inlinehint attribute when it would increase the threshold
109 // and the caller does not need to minimize its size.
110 Function *Callee = CS.getCalledFunction();
111 bool InlineHint = Callee && !Callee->isDeclaration() &&
112 Callee->hasFnAttribute(Attribute::InlineHint);
113 if (InlineHint && Params.HintThreshold && Params.HintThreshold > Thres
114 && !Caller->hasFnAttribute(Attribute::MinSize))
115 Thres = Params.HintThreshold.getValue();
116
117 const DataLayout &DL = Caller->getParent()->getDataLayout();
118 if (!Callee)
119 return (unsigned)Thres;
120
121 const AMDGPUAS AS = AMDGPU::getAMDGPUAS(*Caller->getParent());
122
123 // If we have a pointer to private array passed into a function
124 // it will not be optimized out, leaving scratch usage.
125 // Increase the inline threshold to allow inliniting in this case.
126 uint64_t AllocaSize = 0;
127 SmallPtrSet<const AllocaInst *, 8> AIVisited;
128 for (Value *PtrArg : CS.args()) {
129 Type *Ty = PtrArg->getType();
130 if (!Ty->isPointerTy() ||
131 Ty->getPointerAddressSpace() != AS.PRIVATE_ADDRESS)
132 continue;
133 PtrArg = GetUnderlyingObject(PtrArg, DL);
134 if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) {
135 if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second)
136 continue;
137 AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType());
138 // If the amount of stack memory is excessive we will not be able
139 // to get rid of the scratch anyway, bail out.
140 if (AllocaSize > ArgAllocaCutoff) {
141 AllocaSize = 0;
142 break;
143 }
144 }
145 }
146 if (AllocaSize)
147 Thres += ArgAllocaCost;
148
149 return (unsigned)Thres;
150 }
151
152 // Check if call is just a wrapper around another call.
153 // In this case we only have call and ret instructions.
isWrapperOnlyCall(CallSite CS)154 static bool isWrapperOnlyCall(CallSite CS) {
155 Function *Callee = CS.getCalledFunction();
156 if (!Callee || Callee->size() != 1)
157 return false;
158 const BasicBlock &BB = Callee->getEntryBlock();
159 if (const Instruction *I = BB.getFirstNonPHI()) {
160 if (!isa<CallInst>(I)) {
161 return false;
162 }
163 if (isa<ReturnInst>(*std::next(I->getIterator()))) {
164 LLVM_DEBUG(dbgs() << " Wrapper only call detected: "
165 << Callee->getName() << '\n');
166 return true;
167 }
168 }
169 return false;
170 }
171
getInlineCost(CallSite CS)172 InlineCost AMDGPUInliner::getInlineCost(CallSite CS) {
173 Function *Callee = CS.getCalledFunction();
174 Function *Caller = CS.getCaller();
175 TargetTransformInfo &TTI = TTIWP->getTTI(*Callee);
176
177 if (!Callee || Callee->isDeclaration() || CS.isNoInline() ||
178 !TTI.areInlineCompatible(Caller, Callee))
179 return llvm::InlineCost::getNever();
180
181 if (CS.hasFnAttr(Attribute::AlwaysInline)) {
182 if (isInlineViable(*Callee))
183 return llvm::InlineCost::getAlways();
184 return llvm::InlineCost::getNever();
185 }
186
187 if (isWrapperOnlyCall(CS))
188 return llvm::InlineCost::getAlways();
189
190 InlineParams LocalParams = Params;
191 LocalParams.DefaultThreshold = (int)getInlineThreshold(CS);
192 bool RemarksEnabled = false;
193 const auto &BBs = Caller->getBasicBlockList();
194 if (!BBs.empty()) {
195 auto DI = OptimizationRemark(DEBUG_TYPE, "", DebugLoc(), &BBs.front());
196 if (DI.isEnabled())
197 RemarksEnabled = true;
198 }
199
200 OptimizationRemarkEmitter ORE(Caller);
201 std::function<AssumptionCache &(Function &)> GetAssumptionCache =
202 [this](Function &F) -> AssumptionCache & {
203 return ACT->getAssumptionCache(F);
204 };
205
206 return llvm::getInlineCost(CS, Callee, LocalParams, TTI, GetAssumptionCache,
207 None, PSI, RemarksEnabled ? &ORE : nullptr);
208 }
209