1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass lowers instrprof_increment intrinsics emitted by a frontend for
11 // profiling. It also builds the data structures and initialization code needed
12 // for updating execution counts and emitting the profile at runtime.
13 //
14 //===----------------------------------------------------------------------===//
15
16 #include "llvm/Transforms/Instrumentation.h"
17
18 #include "llvm/ADT/Triple.h"
19 #include "llvm/IR/IRBuilder.h"
20 #include "llvm/IR/IntrinsicInst.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
23
24 using namespace llvm;
25
26 #define DEBUG_TYPE "instrprof"
27
28 namespace {
29
30 class InstrProfiling : public ModulePass {
31 public:
32 static char ID;
33
InstrProfiling()34 InstrProfiling() : ModulePass(ID) {}
35
InstrProfiling(const InstrProfOptions & Options)36 InstrProfiling(const InstrProfOptions &Options)
37 : ModulePass(ID), Options(Options) {}
38
getPassName() const39 const char *getPassName() const override {
40 return "Frontend instrumentation-based coverage lowering";
41 }
42
43 bool runOnModule(Module &M) override;
44
getAnalysisUsage(AnalysisUsage & AU) const45 void getAnalysisUsage(AnalysisUsage &AU) const override {
46 AU.setPreservesCFG();
47 }
48
49 private:
50 InstrProfOptions Options;
51 Module *M;
52 DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
53 std::vector<Value *> UsedVars;
54
isMachO() const55 bool isMachO() const {
56 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
57 }
58
59 /// Get the section name for the counter variables.
getCountersSection() const60 StringRef getCountersSection() const {
61 return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
62 }
63
64 /// Get the section name for the name variables.
getNameSection() const65 StringRef getNameSection() const {
66 return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
67 }
68
69 /// Get the section name for the profile data variables.
getDataSection() const70 StringRef getDataSection() const {
71 return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
72 }
73
74 /// Get the section name for the coverage mapping data.
getCoverageSection() const75 StringRef getCoverageSection() const {
76 return isMachO() ? "__DATA,__llvm_covmap" : "__llvm_covmap";
77 }
78
79 /// Replace instrprof_increment with an increment of the appropriate value.
80 void lowerIncrement(InstrProfIncrementInst *Inc);
81
82 /// Set up the section and uses for coverage data and its references.
83 void lowerCoverageData(GlobalVariable *CoverageData);
84
85 /// Get the region counters for an increment, creating them if necessary.
86 ///
87 /// If the counter array doesn't yet exist, the profile data variables
88 /// referring to them will also be created.
89 GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
90
91 /// Emit runtime registration functions for each profile data variable.
92 void emitRegistration();
93
94 /// Emit the necessary plumbing to pull in the runtime initialization.
95 void emitRuntimeHook();
96
97 /// Add uses of our data variables and runtime hook.
98 void emitUses();
99
100 /// Create a static initializer for our data, on platforms that need it.
101 void emitInitialization();
102 };
103
104 } // anonymous namespace
105
106 char InstrProfiling::ID = 0;
107 INITIALIZE_PASS(InstrProfiling, "instrprof",
108 "Frontend instrumentation-based coverage lowering.", false,
109 false)
110
createInstrProfilingPass(const InstrProfOptions & Options)111 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
112 return new InstrProfiling(Options);
113 }
114
runOnModule(Module & M)115 bool InstrProfiling::runOnModule(Module &M) {
116 bool MadeChange = false;
117
118 this->M = &M;
119 RegionCounters.clear();
120 UsedVars.clear();
121
122 for (Function &F : M)
123 for (BasicBlock &BB : F)
124 for (auto I = BB.begin(), E = BB.end(); I != E;)
125 if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
126 lowerIncrement(Inc);
127 MadeChange = true;
128 }
129 if (GlobalVariable *Coverage = M.getNamedGlobal("__llvm_coverage_mapping")) {
130 lowerCoverageData(Coverage);
131 MadeChange = true;
132 }
133 if (!MadeChange)
134 return false;
135
136 emitRegistration();
137 emitRuntimeHook();
138 emitUses();
139 emitInitialization();
140 return true;
141 }
142
lowerIncrement(InstrProfIncrementInst * Inc)143 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
144 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
145
146 IRBuilder<> Builder(Inc->getParent(), *Inc);
147 uint64_t Index = Inc->getIndex()->getZExtValue();
148 llvm::Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
149 llvm::Value *Count = Builder.CreateLoad(Addr, "pgocount");
150 Count = Builder.CreateAdd(Count, Builder.getInt64(1));
151 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
152 Inc->eraseFromParent();
153 }
154
lowerCoverageData(GlobalVariable * CoverageData)155 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
156 CoverageData->setSection(getCoverageSection());
157 CoverageData->setAlignment(8);
158
159 Constant *Init = CoverageData->getInitializer();
160 // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
161 // for some C. If not, the frontend's given us something broken.
162 assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
163 assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
164 "invalid function list in coverage map");
165 ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
166 for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
167 Constant *Record = Records->getOperand(I);
168 Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
169
170 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
171 GlobalVariable *Name = cast<GlobalVariable>(V);
172
173 // If we have region counters for this name, we've already handled it.
174 auto It = RegionCounters.find(Name);
175 if (It != RegionCounters.end())
176 continue;
177
178 // Move the name variable to the right section.
179 Name->setSection(getNameSection());
180 Name->setAlignment(1);
181 }
182 }
183
184 /// Get the name of a profiling variable for a particular function.
getVarName(InstrProfIncrementInst * Inc,StringRef VarName)185 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
186 auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
187 StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
188 return ("__llvm_profile_" + VarName + "_" + Name).str();
189 }
190
191 GlobalVariable *
getOrCreateRegionCounters(InstrProfIncrementInst * Inc)192 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
193 GlobalVariable *Name = Inc->getName();
194 auto It = RegionCounters.find(Name);
195 if (It != RegionCounters.end())
196 return It->second;
197
198 // Move the name variable to the right section.
199 Name->setSection(getNameSection());
200 Name->setAlignment(1);
201
202 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
203 LLVMContext &Ctx = M->getContext();
204 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
205
206 // Create the counters variable.
207 auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
208 Constant::getNullValue(CounterTy),
209 getVarName(Inc, "counters"));
210 Counters->setVisibility(Name->getVisibility());
211 Counters->setSection(getCountersSection());
212 Counters->setAlignment(8);
213
214 RegionCounters[Inc->getName()] = Counters;
215
216 // Create data variable.
217 auto *NameArrayTy = Name->getType()->getPointerElementType();
218 auto *Int32Ty = Type::getInt32Ty(Ctx);
219 auto *Int64Ty = Type::getInt64Ty(Ctx);
220 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
221 auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
222
223 Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
224 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
225 Constant *DataVals[] = {
226 ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
227 ConstantInt::get(Int32Ty, NumCounters),
228 ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
229 ConstantExpr::getBitCast(Name, Int8PtrTy),
230 ConstantExpr::getBitCast(Counters, Int64PtrTy)};
231 auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
232 ConstantStruct::get(DataTy, DataVals),
233 getVarName(Inc, "data"));
234 Data->setVisibility(Name->getVisibility());
235 Data->setSection(getDataSection());
236 Data->setAlignment(8);
237
238 // Mark the data variable as used so that it isn't stripped out.
239 UsedVars.push_back(Data);
240
241 return Counters;
242 }
243
emitRegistration()244 void InstrProfiling::emitRegistration() {
245 // Don't do this for Darwin. compiler-rt uses linker magic.
246 if (Triple(M->getTargetTriple()).isOSDarwin())
247 return;
248
249 // Construct the function.
250 auto *VoidTy = Type::getVoidTy(M->getContext());
251 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
252 auto *RegisterFTy = FunctionType::get(VoidTy, false);
253 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
254 "__llvm_profile_register_functions", M);
255 RegisterF->setUnnamedAddr(true);
256 if (Options.NoRedZone)
257 RegisterF->addFnAttr(Attribute::NoRedZone);
258
259 auto *RuntimeRegisterTy = llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
260 auto *RuntimeRegisterF =
261 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
262 "__llvm_profile_register_function", M);
263
264 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
265 for (Value *Data : UsedVars)
266 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
267 IRB.CreateRetVoid();
268 }
269
emitRuntimeHook()270 void InstrProfiling::emitRuntimeHook() {
271 const char *const RuntimeVarName = "__llvm_profile_runtime";
272 const char *const RuntimeUserName = "__llvm_profile_runtime_user";
273
274 // If the module's provided its own runtime, we don't need to do anything.
275 if (M->getGlobalVariable(RuntimeVarName))
276 return;
277
278 // Declare an external variable that will pull in the runtime initialization.
279 auto *Int32Ty = Type::getInt32Ty(M->getContext());
280 auto *Var =
281 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
282 nullptr, RuntimeVarName);
283
284 // Make a function that uses it.
285 auto *User =
286 Function::Create(FunctionType::get(Int32Ty, false),
287 GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
288 User->addFnAttr(Attribute::NoInline);
289 if (Options.NoRedZone)
290 User->addFnAttr(Attribute::NoRedZone);
291 User->setVisibility(GlobalValue::HiddenVisibility);
292
293 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
294 auto *Load = IRB.CreateLoad(Var);
295 IRB.CreateRet(Load);
296
297 // Mark the user variable as used so that it isn't stripped out.
298 UsedVars.push_back(User);
299 }
300
emitUses()301 void InstrProfiling::emitUses() {
302 if (UsedVars.empty())
303 return;
304
305 GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
306 std::vector<Constant*> MergedVars;
307 if (LLVMUsed) {
308 // Collect the existing members of llvm.used.
309 ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
310 for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
311 MergedVars.push_back(Inits->getOperand(I));
312 LLVMUsed->eraseFromParent();
313 }
314
315 Type *i8PTy = Type::getInt8PtrTy(M->getContext());
316 // Add uses for our data.
317 for (auto *Value : UsedVars)
318 MergedVars.push_back(
319 ConstantExpr::getBitCast(cast<llvm::Constant>(Value), i8PTy));
320
321 // Recreate llvm.used.
322 ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
323 LLVMUsed = new llvm::GlobalVariable(
324 *M, ATy, false, llvm::GlobalValue::AppendingLinkage,
325 llvm::ConstantArray::get(ATy, MergedVars), "llvm.used");
326
327 LLVMUsed->setSection("llvm.metadata");
328 }
329
emitInitialization()330 void InstrProfiling::emitInitialization() {
331 Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
332 if (!RegisterF)
333 return;
334
335 // Create the initialization function.
336 auto *VoidTy = Type::getVoidTy(M->getContext());
337 auto *F =
338 Function::Create(FunctionType::get(VoidTy, false),
339 GlobalValue::InternalLinkage, "__llvm_profile_init", M);
340 F->setUnnamedAddr(true);
341 F->addFnAttr(Attribute::NoInline);
342 if (Options.NoRedZone)
343 F->addFnAttr(Attribute::NoRedZone);
344
345 // Add the basic block and the necessary calls.
346 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
347 IRB.CreateCall(RegisterF);
348 IRB.CreateRetVoid();
349
350 appendToGlobalCtors(*M, F, 0);
351 }
352