1 //=-- SampleProf.cpp - Sample profiling format support --------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains common definitions used in the reading and writing of
10 // sample profile data.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/ProfileData/SampleProf.h"
15 #include "llvm/Config/llvm-config.h"
16 #include "llvm/IR/DebugInfoMetadata.h"
17 #include "llvm/ProfileData/SampleProfReader.h"
18 #include "llvm/Support/Compiler.h"
19 #include "llvm/Support/Debug.h"
20 #include "llvm/Support/Error.h"
21 #include "llvm/Support/ErrorHandling.h"
22 #include "llvm/Support/LEB128.h"
23 #include "llvm/Support/ManagedStatic.h"
24 #include "llvm/Support/raw_ostream.h"
25 #include <string>
26 #include <system_error>
27
28 using namespace llvm;
29 using namespace sampleprof;
30
31 namespace llvm {
32 namespace sampleprof {
33 SampleProfileFormat FunctionSamples::Format;
34 bool FunctionSamples::ProfileIsCS = false;
35 bool FunctionSamples::UseMD5;
36 } // namespace sampleprof
37 } // namespace llvm
38
39 namespace {
40
41 // FIXME: This class is only here to support the transition to llvm::Error. It
42 // will be removed once this transition is complete. Clients should prefer to
43 // deal with the Error value directly, rather than converting to error_code.
44 class SampleProfErrorCategoryType : public std::error_category {
name() const45 const char *name() const noexcept override { return "llvm.sampleprof"; }
46
message(int IE) const47 std::string message(int IE) const override {
48 sampleprof_error E = static_cast<sampleprof_error>(IE);
49 switch (E) {
50 case sampleprof_error::success:
51 return "Success";
52 case sampleprof_error::bad_magic:
53 return "Invalid sample profile data (bad magic)";
54 case sampleprof_error::unsupported_version:
55 return "Unsupported sample profile format version";
56 case sampleprof_error::too_large:
57 return "Too much profile data";
58 case sampleprof_error::truncated:
59 return "Truncated profile data";
60 case sampleprof_error::malformed:
61 return "Malformed sample profile data";
62 case sampleprof_error::unrecognized_format:
63 return "Unrecognized sample profile encoding format";
64 case sampleprof_error::unsupported_writing_format:
65 return "Profile encoding format unsupported for writing operations";
66 case sampleprof_error::truncated_name_table:
67 return "Truncated function name table";
68 case sampleprof_error::not_implemented:
69 return "Unimplemented feature";
70 case sampleprof_error::counter_overflow:
71 return "Counter overflow";
72 case sampleprof_error::ostream_seek_unsupported:
73 return "Ostream does not support seek";
74 case sampleprof_error::compress_failed:
75 return "Compress failure";
76 case sampleprof_error::uncompress_failed:
77 return "Uncompress failure";
78 case sampleprof_error::zlib_unavailable:
79 return "Zlib is unavailable";
80 }
81 llvm_unreachable("A value of sampleprof_error has no message.");
82 }
83 };
84
85 } // end anonymous namespace
86
87 static ManagedStatic<SampleProfErrorCategoryType> ErrorCategory;
88
sampleprof_category()89 const std::error_category &llvm::sampleprof_category() {
90 return *ErrorCategory;
91 }
92
print(raw_ostream & OS) const93 void LineLocation::print(raw_ostream &OS) const {
94 OS << LineOffset;
95 if (Discriminator > 0)
96 OS << "." << Discriminator;
97 }
98
operator <<(raw_ostream & OS,const LineLocation & Loc)99 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
100 const LineLocation &Loc) {
101 Loc.print(OS);
102 return OS;
103 }
104
105 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const106 LLVM_DUMP_METHOD void LineLocation::dump() const { print(dbgs()); }
107 #endif
108
109 /// Print the sample record to the stream \p OS indented by \p Indent.
print(raw_ostream & OS,unsigned Indent) const110 void SampleRecord::print(raw_ostream &OS, unsigned Indent) const {
111 OS << NumSamples;
112 if (hasCalls()) {
113 OS << ", calls:";
114 for (const auto &I : getSortedCallTargets())
115 OS << " " << I.first << ":" << I.second;
116 }
117 OS << "\n";
118 }
119
120 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const121 LLVM_DUMP_METHOD void SampleRecord::dump() const { print(dbgs(), 0); }
122 #endif
123
operator <<(raw_ostream & OS,const SampleRecord & Sample)124 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
125 const SampleRecord &Sample) {
126 Sample.print(OS, 0);
127 return OS;
128 }
129
130 /// Print the samples collected for a function on stream \p OS.
print(raw_ostream & OS,unsigned Indent) const131 void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
132 OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size()
133 << " sampled lines\n";
134
135 OS.indent(Indent);
136 if (!BodySamples.empty()) {
137 OS << "Samples collected in the function's body {\n";
138 SampleSorter<LineLocation, SampleRecord> SortedBodySamples(BodySamples);
139 for (const auto &SI : SortedBodySamples.get()) {
140 OS.indent(Indent + 2);
141 OS << SI->first << ": " << SI->second;
142 }
143 OS.indent(Indent);
144 OS << "}\n";
145 } else {
146 OS << "No samples collected in the function's body\n";
147 }
148
149 OS.indent(Indent);
150 if (!CallsiteSamples.empty()) {
151 OS << "Samples collected in inlined callsites {\n";
152 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
153 CallsiteSamples);
154 for (const auto &CS : SortedCallsiteSamples.get()) {
155 for (const auto &FS : CS->second) {
156 OS.indent(Indent + 2);
157 OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
158 FS.second.print(OS, Indent + 4);
159 }
160 }
161 OS.indent(Indent);
162 OS << "}\n";
163 } else {
164 OS << "No inlined callsites in this function\n";
165 }
166 }
167
operator <<(raw_ostream & OS,const FunctionSamples & FS)168 raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS,
169 const FunctionSamples &FS) {
170 FS.print(OS);
171 return OS;
172 }
173
getOffset(const DILocation * DIL)174 unsigned FunctionSamples::getOffset(const DILocation *DIL) {
175 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
176 0xffff;
177 }
178
findFunctionSamples(const DILocation * DIL,SampleProfileReaderItaniumRemapper * Remapper) const179 const FunctionSamples *FunctionSamples::findFunctionSamples(
180 const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
181 assert(DIL);
182 SmallVector<std::pair<LineLocation, StringRef>, 10> S;
183
184 const DILocation *PrevDIL = DIL;
185 for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
186 S.push_back(std::make_pair(
187 LineLocation(getOffset(DIL), DIL->getBaseDiscriminator()),
188 PrevDIL->getScope()->getSubprogram()->getLinkageName()));
189 PrevDIL = DIL;
190 }
191 if (S.size() == 0)
192 return this;
193 const FunctionSamples *FS = this;
194 for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) {
195 FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper);
196 }
197 return FS;
198 }
199
findAllNames(DenseSet<StringRef> & NameSet) const200 void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
201 NameSet.insert(Name);
202 for (const auto &BS : BodySamples)
203 for (const auto &TS : BS.second.getCallTargets())
204 NameSet.insert(TS.getKey());
205
206 for (const auto &CS : CallsiteSamples) {
207 for (const auto &NameFS : CS.second) {
208 NameSet.insert(NameFS.first);
209 NameFS.second.findAllNames(NameSet);
210 }
211 }
212 }
213
findFunctionSamplesAt(const LineLocation & Loc,StringRef CalleeName,SampleProfileReaderItaniumRemapper * Remapper) const214 const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
215 const LineLocation &Loc, StringRef CalleeName,
216 SampleProfileReaderItaniumRemapper *Remapper) const {
217 std::string CalleeGUID;
218 CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
219
220 auto iter = CallsiteSamples.find(Loc);
221 if (iter == CallsiteSamples.end())
222 return nullptr;
223 auto FS = iter->second.find(CalleeName);
224 if (FS != iter->second.end())
225 return &FS->second;
226 if (Remapper) {
227 if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
228 auto FS = iter->second.find(*NameInProfile);
229 if (FS != iter->second.end())
230 return &FS->second;
231 }
232 }
233 // If we cannot find exact match of the callee name, return the FS with
234 // the max total count. Only do this when CalleeName is not provided,
235 // i.e., only for indirect calls.
236 if (!CalleeName.empty())
237 return nullptr;
238 uint64_t MaxTotalSamples = 0;
239 const FunctionSamples *R = nullptr;
240 for (const auto &NameFS : iter->second)
241 if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
242 MaxTotalSamples = NameFS.second.getTotalSamples();
243 R = &NameFS.second;
244 }
245 return R;
246 }
247
248 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const249 LLVM_DUMP_METHOD void FunctionSamples::dump() const { print(dbgs(), 0); }
250 #endif
251
read(const uint8_t * Data,uint64_t ListSize)252 std::error_code ProfileSymbolList::read(const uint8_t *Data,
253 uint64_t ListSize) {
254 const char *ListStart = reinterpret_cast<const char *>(Data);
255 uint64_t Size = 0;
256 while (Size < ListSize) {
257 StringRef Str(ListStart + Size);
258 add(Str);
259 Size += Str.size() + 1;
260 }
261 if (Size != ListSize)
262 return sampleprof_error::malformed;
263 return sampleprof_error::success;
264 }
265
write(raw_ostream & OS)266 std::error_code ProfileSymbolList::write(raw_ostream &OS) {
267 // Sort the symbols before output. If doing compression.
268 // It will make the compression much more effective.
269 std::vector<StringRef> SortedList;
270 SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end());
271 llvm::sort(SortedList);
272
273 std::string OutputString;
274 for (auto &Sym : SortedList) {
275 OutputString.append(Sym.str());
276 OutputString.append(1, '\0');
277 }
278
279 OS << OutputString;
280 return sampleprof_error::success;
281 }
282
dump(raw_ostream & OS) const283 void ProfileSymbolList::dump(raw_ostream &OS) const {
284 OS << "======== Dump profile symbol list ========\n";
285 std::vector<StringRef> SortedList;
286 SortedList.insert(SortedList.begin(), Syms.begin(), Syms.end());
287 llvm::sort(SortedList);
288
289 for (auto &Sym : SortedList)
290 OS << Sym << "\n";
291 }
292