1 //===- BlockFrequencyInfo.cpp - Block Frequency Analysis ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Loops should be simplified before this analysis.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Analysis/BlockFrequencyInfo.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/None.h"
16 #include "llvm/ADT/iterator.h"
17 #include "llvm/Analysis/BlockFrequencyInfoImpl.h"
18 #include "llvm/Analysis/BranchProbabilityInfo.h"
19 #include "llvm/Analysis/LoopInfo.h"
20 #include "llvm/IR/CFG.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/PassManager.h"
23 #include "llvm/InitializePasses.h"
24 #include "llvm/Pass.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/GraphWriter.h"
27 #include "llvm/Support/raw_ostream.h"
28 #include <algorithm>
29 #include <cassert>
30 #include <string>
31
32 using namespace llvm;
33
34 #define DEBUG_TYPE "block-freq"
35
36 static cl::opt<GVDAGType> ViewBlockFreqPropagationDAG(
37 "view-block-freq-propagation-dags", cl::Hidden,
38 cl::desc("Pop up a window to show a dag displaying how block "
39 "frequencies propagation through the CFG."),
40 cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."),
41 clEnumValN(GVDT_Fraction, "fraction",
42 "display a graph using the "
43 "fractional block frequency representation."),
44 clEnumValN(GVDT_Integer, "integer",
45 "display a graph using the raw "
46 "integer fractional block frequency representation."),
47 clEnumValN(GVDT_Count, "count", "display a graph using the real "
48 "profile count if available.")));
49
50 cl::opt<std::string>
51 ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden,
52 cl::desc("The option to specify "
53 "the name of the function "
54 "whose CFG will be displayed."));
55
56 cl::opt<unsigned>
57 ViewHotFreqPercent("view-hot-freq-percent", cl::init(10), cl::Hidden,
58 cl::desc("An integer in percent used to specify "
59 "the hot blocks/edges to be displayed "
60 "in red: a block or edge whose frequency "
61 "is no less than the max frequency of the "
62 "function multiplied by this percent."));
63
64 // Command line option to turn on CFG dot or text dump after profile annotation.
65 cl::opt<PGOViewCountsType> PGOViewCounts(
66 "pgo-view-counts", cl::Hidden,
67 cl::desc("A boolean option to show CFG dag or text with "
68 "block profile counts and branch probabilities "
69 "right after PGO profile annotation step. The "
70 "profile counts are computed using branch "
71 "probabilities from the runtime profile data and "
72 "block frequency propagation algorithm. To view "
73 "the raw counts from the profile, use option "
74 "-pgo-view-raw-counts instead. To limit graph "
75 "display to only one function, use filtering option "
76 "-view-bfi-func-name."),
77 cl::values(clEnumValN(PGOVCT_None, "none", "do not show."),
78 clEnumValN(PGOVCT_Graph, "graph", "show a graph."),
79 clEnumValN(PGOVCT_Text, "text", "show in text.")));
80
81 static cl::opt<bool> PrintBlockFreq(
82 "print-bfi", cl::init(false), cl::Hidden,
83 cl::desc("Print the block frequency info."));
84
85 cl::opt<std::string> PrintBlockFreqFuncName(
86 "print-bfi-func-name", cl::Hidden,
87 cl::desc("The option to specify the name of the function "
88 "whose block frequency info is printed."));
89
90 namespace llvm {
91
getGVDT()92 static GVDAGType getGVDT() {
93 if (PGOViewCounts == PGOVCT_Graph)
94 return GVDT_Count;
95 return ViewBlockFreqPropagationDAG;
96 }
97
98 template <>
99 struct GraphTraits<BlockFrequencyInfo *> {
100 using NodeRef = const BasicBlock *;
101 using ChildIteratorType = const_succ_iterator;
102 using nodes_iterator = pointer_iterator<Function::const_iterator>;
103
getEntryNodellvm::GraphTraits104 static NodeRef getEntryNode(const BlockFrequencyInfo *G) {
105 return &G->getFunction()->front();
106 }
107
child_beginllvm::GraphTraits108 static ChildIteratorType child_begin(const NodeRef N) {
109 return succ_begin(N);
110 }
111
child_endllvm::GraphTraits112 static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
113
nodes_beginllvm::GraphTraits114 static nodes_iterator nodes_begin(const BlockFrequencyInfo *G) {
115 return nodes_iterator(G->getFunction()->begin());
116 }
117
nodes_endllvm::GraphTraits118 static nodes_iterator nodes_end(const BlockFrequencyInfo *G) {
119 return nodes_iterator(G->getFunction()->end());
120 }
121 };
122
123 using BFIDOTGTraitsBase =
124 BFIDOTGraphTraitsBase<BlockFrequencyInfo, BranchProbabilityInfo>;
125
126 template <>
127 struct DOTGraphTraits<BlockFrequencyInfo *> : public BFIDOTGTraitsBase {
DOTGraphTraitsllvm::DOTGraphTraits128 explicit DOTGraphTraits(bool isSimple = false)
129 : BFIDOTGTraitsBase(isSimple) {}
130
getNodeLabelllvm::DOTGraphTraits131 std::string getNodeLabel(const BasicBlock *Node,
132 const BlockFrequencyInfo *Graph) {
133
134 return BFIDOTGTraitsBase::getNodeLabel(Node, Graph, getGVDT());
135 }
136
getNodeAttributesllvm::DOTGraphTraits137 std::string getNodeAttributes(const BasicBlock *Node,
138 const BlockFrequencyInfo *Graph) {
139 return BFIDOTGTraitsBase::getNodeAttributes(Node, Graph,
140 ViewHotFreqPercent);
141 }
142
getEdgeAttributesllvm::DOTGraphTraits143 std::string getEdgeAttributes(const BasicBlock *Node, EdgeIter EI,
144 const BlockFrequencyInfo *BFI) {
145 return BFIDOTGTraitsBase::getEdgeAttributes(Node, EI, BFI, BFI->getBPI(),
146 ViewHotFreqPercent);
147 }
148 };
149
150 } // end namespace llvm
151
152 BlockFrequencyInfo::BlockFrequencyInfo() = default;
153
BlockFrequencyInfo(const Function & F,const BranchProbabilityInfo & BPI,const LoopInfo & LI)154 BlockFrequencyInfo::BlockFrequencyInfo(const Function &F,
155 const BranchProbabilityInfo &BPI,
156 const LoopInfo &LI) {
157 calculate(F, BPI, LI);
158 }
159
BlockFrequencyInfo(BlockFrequencyInfo && Arg)160 BlockFrequencyInfo::BlockFrequencyInfo(BlockFrequencyInfo &&Arg)
161 : BFI(std::move(Arg.BFI)) {}
162
operator =(BlockFrequencyInfo && RHS)163 BlockFrequencyInfo &BlockFrequencyInfo::operator=(BlockFrequencyInfo &&RHS) {
164 releaseMemory();
165 BFI = std::move(RHS.BFI);
166 return *this;
167 }
168
169 // Explicitly define the default constructor otherwise it would be implicitly
170 // defined at the first ODR-use which is the BFI member in the
171 // LazyBlockFrequencyInfo header. The dtor needs the BlockFrequencyInfoImpl
172 // template instantiated which is not available in the header.
173 BlockFrequencyInfo::~BlockFrequencyInfo() = default;
174
invalidate(Function & F,const PreservedAnalyses & PA,FunctionAnalysisManager::Invalidator &)175 bool BlockFrequencyInfo::invalidate(Function &F, const PreservedAnalyses &PA,
176 FunctionAnalysisManager::Invalidator &) {
177 // Check whether the analysis, all analyses on functions, or the function's
178 // CFG have been preserved.
179 auto PAC = PA.getChecker<BlockFrequencyAnalysis>();
180 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>() ||
181 PAC.preservedSet<CFGAnalyses>());
182 }
183
calculate(const Function & F,const BranchProbabilityInfo & BPI,const LoopInfo & LI)184 void BlockFrequencyInfo::calculate(const Function &F,
185 const BranchProbabilityInfo &BPI,
186 const LoopInfo &LI) {
187 if (!BFI)
188 BFI.reset(new ImplType);
189 BFI->calculate(F, BPI, LI);
190 if (ViewBlockFreqPropagationDAG != GVDT_None &&
191 (ViewBlockFreqFuncName.empty() ||
192 F.getName().equals(ViewBlockFreqFuncName))) {
193 view();
194 }
195 if (PrintBlockFreq &&
196 (PrintBlockFreqFuncName.empty() ||
197 F.getName().equals(PrintBlockFreqFuncName))) {
198 print(dbgs());
199 }
200 }
201
getBlockFreq(const BasicBlock * BB) const202 BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const {
203 return BFI ? BFI->getBlockFreq(BB) : 0;
204 }
205
206 Optional<uint64_t>
getBlockProfileCount(const BasicBlock * BB,bool AllowSynthetic) const207 BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB,
208 bool AllowSynthetic) const {
209 if (!BFI)
210 return None;
211
212 return BFI->getBlockProfileCount(*getFunction(), BB, AllowSynthetic);
213 }
214
215 Optional<uint64_t>
getProfileCountFromFreq(uint64_t Freq) const216 BlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const {
217 if (!BFI)
218 return None;
219 return BFI->getProfileCountFromFreq(*getFunction(), Freq);
220 }
221
isIrrLoopHeader(const BasicBlock * BB)222 bool BlockFrequencyInfo::isIrrLoopHeader(const BasicBlock *BB) {
223 assert(BFI && "Expected analysis to be available");
224 return BFI->isIrrLoopHeader(BB);
225 }
226
setBlockFreq(const BasicBlock * BB,uint64_t Freq)227 void BlockFrequencyInfo::setBlockFreq(const BasicBlock *BB, uint64_t Freq) {
228 assert(BFI && "Expected analysis to be available");
229 BFI->setBlockFreq(BB, Freq);
230 }
231
setBlockFreqAndScale(const BasicBlock * ReferenceBB,uint64_t Freq,SmallPtrSetImpl<BasicBlock * > & BlocksToScale)232 void BlockFrequencyInfo::setBlockFreqAndScale(
233 const BasicBlock *ReferenceBB, uint64_t Freq,
234 SmallPtrSetImpl<BasicBlock *> &BlocksToScale) {
235 assert(BFI && "Expected analysis to be available");
236 // Use 128 bits APInt to avoid overflow.
237 APInt NewFreq(128, Freq);
238 APInt OldFreq(128, BFI->getBlockFreq(ReferenceBB).getFrequency());
239 APInt BBFreq(128, 0);
240 for (auto *BB : BlocksToScale) {
241 BBFreq = BFI->getBlockFreq(BB).getFrequency();
242 // Multiply first by NewFreq and then divide by OldFreq
243 // to minimize loss of precision.
244 BBFreq *= NewFreq;
245 // udiv is an expensive operation in the general case. If this ends up being
246 // a hot spot, one of the options proposed in
247 // https://reviews.llvm.org/D28535#650071 could be used to avoid this.
248 BBFreq = BBFreq.udiv(OldFreq);
249 BFI->setBlockFreq(BB, BBFreq.getLimitedValue());
250 }
251 BFI->setBlockFreq(ReferenceBB, Freq);
252 }
253
254 /// Pop up a ghostview window with the current block frequency propagation
255 /// rendered using dot.
view(StringRef title) const256 void BlockFrequencyInfo::view(StringRef title) const {
257 ViewGraph(const_cast<BlockFrequencyInfo *>(this), title);
258 }
259
getFunction() const260 const Function *BlockFrequencyInfo::getFunction() const {
261 return BFI ? BFI->getFunction() : nullptr;
262 }
263
getBPI() const264 const BranchProbabilityInfo *BlockFrequencyInfo::getBPI() const {
265 return BFI ? &BFI->getBPI() : nullptr;
266 }
267
268 raw_ostream &BlockFrequencyInfo::
printBlockFreq(raw_ostream & OS,const BlockFrequency Freq) const269 printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const {
270 return BFI ? BFI->printBlockFreq(OS, Freq) : OS;
271 }
272
273 raw_ostream &
printBlockFreq(raw_ostream & OS,const BasicBlock * BB) const274 BlockFrequencyInfo::printBlockFreq(raw_ostream &OS,
275 const BasicBlock *BB) const {
276 return BFI ? BFI->printBlockFreq(OS, BB) : OS;
277 }
278
getEntryFreq() const279 uint64_t BlockFrequencyInfo::getEntryFreq() const {
280 return BFI ? BFI->getEntryFreq() : 0;
281 }
282
releaseMemory()283 void BlockFrequencyInfo::releaseMemory() { BFI.reset(); }
284
print(raw_ostream & OS) const285 void BlockFrequencyInfo::print(raw_ostream &OS) const {
286 if (BFI)
287 BFI->print(OS);
288 }
289
verifyMatch(BlockFrequencyInfo & Other) const290 void BlockFrequencyInfo::verifyMatch(BlockFrequencyInfo &Other) const {
291 if (BFI)
292 BFI->verifyMatch(*Other.BFI);
293 }
294
295 INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq",
296 "Block Frequency Analysis", true, true)
297 INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
298 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
299 INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
300 "Block Frequency Analysis", true, true)
301
302 char BlockFrequencyInfoWrapperPass::ID = 0;
303
BlockFrequencyInfoWrapperPass()304 BlockFrequencyInfoWrapperPass::BlockFrequencyInfoWrapperPass()
305 : FunctionPass(ID) {
306 initializeBlockFrequencyInfoWrapperPassPass(*PassRegistry::getPassRegistry());
307 }
308
309 BlockFrequencyInfoWrapperPass::~BlockFrequencyInfoWrapperPass() = default;
310
print(raw_ostream & OS,const Module *) const311 void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS,
312 const Module *) const {
313 BFI.print(OS);
314 }
315
getAnalysisUsage(AnalysisUsage & AU) const316 void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
317 AU.addRequired<BranchProbabilityInfoWrapperPass>();
318 AU.addRequired<LoopInfoWrapperPass>();
319 AU.setPreservesAll();
320 }
321
releaseMemory()322 void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); }
323
runOnFunction(Function & F)324 bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) {
325 BranchProbabilityInfo &BPI =
326 getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
327 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
328 BFI.calculate(F, BPI, LI);
329 return false;
330 }
331
332 AnalysisKey BlockFrequencyAnalysis::Key;
run(Function & F,FunctionAnalysisManager & AM)333 BlockFrequencyInfo BlockFrequencyAnalysis::run(Function &F,
334 FunctionAnalysisManager &AM) {
335 BlockFrequencyInfo BFI;
336 BFI.calculate(F, AM.getResult<BranchProbabilityAnalysis>(F),
337 AM.getResult<LoopAnalysis>(F));
338 return BFI;
339 }
340
341 PreservedAnalyses
run(Function & F,FunctionAnalysisManager & AM)342 BlockFrequencyPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
343 OS << "Printing analysis results of BFI for function "
344 << "'" << F.getName() << "':"
345 << "\n";
346 AM.getResult<BlockFrequencyAnalysis>(F).print(OS);
347 return PreservedAnalyses::all();
348 }
349