• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //=-- InstrProf.cpp - Instrumented profiling format support -----------------=//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains support for clang's instrumentation based PGO and
11 // coverage.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/ProfileData/InstrProf.h"
16 #include "llvm/ADT/StringExtras.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/Function.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/MDBuilder.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Support/Compression.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/LEB128.h"
25 #include "llvm/Support/ManagedStatic.h"
26 #include "llvm/Support/Path.h"
27 
28 using namespace llvm;
29 
30 static cl::opt<bool> StaticFuncFullModulePrefix(
31     "static-func-full-module-prefix", cl::init(false),
32     cl::desc("Use full module build paths in the profile counter names for "
33              "static functions."));
34 
35 namespace {
getInstrProfErrString(instrprof_error Err)36 std::string getInstrProfErrString(instrprof_error Err) {
37   switch (Err) {
38   case instrprof_error::success:
39     return "Success";
40   case instrprof_error::eof:
41     return "End of File";
42   case instrprof_error::unrecognized_format:
43     return "Unrecognized instrumentation profile encoding format";
44   case instrprof_error::bad_magic:
45     return "Invalid instrumentation profile data (bad magic)";
46   case instrprof_error::bad_header:
47     return "Invalid instrumentation profile data (file header is corrupt)";
48   case instrprof_error::unsupported_version:
49     return "Unsupported instrumentation profile format version";
50   case instrprof_error::unsupported_hash_type:
51     return "Unsupported instrumentation profile hash type";
52   case instrprof_error::too_large:
53     return "Too much profile data";
54   case instrprof_error::truncated:
55     return "Truncated profile data";
56   case instrprof_error::malformed:
57     return "Malformed instrumentation profile data";
58   case instrprof_error::unknown_function:
59     return "No profile data available for function";
60   case instrprof_error::hash_mismatch:
61     return "Function control flow change detected (hash mismatch)";
62   case instrprof_error::count_mismatch:
63     return "Function basic block count change detected (counter mismatch)";
64   case instrprof_error::counter_overflow:
65     return "Counter overflow";
66   case instrprof_error::value_site_count_mismatch:
67     return "Function value site count change detected (counter mismatch)";
68   case instrprof_error::compress_failed:
69     return "Failed to compress data (zlib)";
70   case instrprof_error::uncompress_failed:
71     return "Failed to uncompress data (zlib)";
72   }
73   llvm_unreachable("A value of instrprof_error has no message.");
74 }
75 
76 // FIXME: This class is only here to support the transition to llvm::Error. It
77 // will be removed once this transition is complete. Clients should prefer to
78 // deal with the Error value directly, rather than converting to error_code.
79 class InstrProfErrorCategoryType : public std::error_category {
name() const80   const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; }
message(int IE) const81   std::string message(int IE) const override {
82     return getInstrProfErrString(static_cast<instrprof_error>(IE));
83   }
84 };
85 } // end anonymous namespace
86 
87 static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
88 
instrprof_category()89 const std::error_category &llvm::instrprof_category() {
90   return *ErrorCategory;
91 }
92 
93 namespace llvm {
94 
addError(instrprof_error IE)95 void SoftInstrProfErrors::addError(instrprof_error IE) {
96   if (IE == instrprof_error::success)
97     return;
98 
99   if (FirstError == instrprof_error::success)
100     FirstError = IE;
101 
102   switch (IE) {
103   case instrprof_error::hash_mismatch:
104     ++NumHashMismatches;
105     break;
106   case instrprof_error::count_mismatch:
107     ++NumCountMismatches;
108     break;
109   case instrprof_error::counter_overflow:
110     ++NumCounterOverflows;
111     break;
112   case instrprof_error::value_site_count_mismatch:
113     ++NumValueSiteCountMismatches;
114     break;
115   default:
116     llvm_unreachable("Not a soft error");
117   }
118 }
119 
message() const120 std::string InstrProfError::message() const {
121   return getInstrProfErrString(Err);
122 }
123 
124 char InstrProfError::ID = 0;
125 
getPGOFuncName(StringRef RawFuncName,GlobalValue::LinkageTypes Linkage,StringRef FileName,uint64_t Version LLVM_ATTRIBUTE_UNUSED)126 std::string getPGOFuncName(StringRef RawFuncName,
127                            GlobalValue::LinkageTypes Linkage,
128                            StringRef FileName,
129                            uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
130   return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
131 }
132 
133 // Return the PGOFuncName. This function has some special handling when called
134 // in LTO optimization. The following only applies when calling in LTO passes
135 // (when \c InLTO is true): LTO's internalization privatizes many global linkage
136 // symbols. This happens after value profile annotation, but those internal
137 // linkage functions should not have a source prefix.
138 // To differentiate compiler generated internal symbols from original ones,
139 // PGOFuncName meta data are created and attached to the original internal
140 // symbols in the value profile annotation step
141 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
142 // data, its original linkage must be non-internal.
getPGOFuncName(const Function & F,bool InLTO,uint64_t Version)143 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
144   if (!InLTO) {
145     StringRef FileName = (StaticFuncFullModulePrefix
146                               ? F.getParent()->getName()
147                               : sys::path::filename(F.getParent()->getName()));
148     return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
149   }
150 
151   // In LTO mode (when InLTO is true), first check if there is a meta data.
152   if (MDNode *MD = getPGOFuncNameMetadata(F)) {
153     StringRef S = cast<MDString>(MD->getOperand(0))->getString();
154     return S.str();
155   }
156 
157   // If there is no meta data, the function must be a global before the value
158   // profile annotation pass. Its current linkage may be internal if it is
159   // internalized in LTO mode.
160   return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
161 }
162 
getFuncNameWithoutPrefix(StringRef PGOFuncName,StringRef FileName)163 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
164   if (FileName.empty())
165     return PGOFuncName;
166   // Drop the file name including ':'. See also getPGOFuncName.
167   if (PGOFuncName.startswith(FileName))
168     PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
169   return PGOFuncName;
170 }
171 
172 // \p FuncName is the string used as profile lookup key for the function. A
173 // symbol is created to hold the name. Return the legalized symbol name.
getPGOFuncNameVarName(StringRef FuncName,GlobalValue::LinkageTypes Linkage)174 std::string getPGOFuncNameVarName(StringRef FuncName,
175                                   GlobalValue::LinkageTypes Linkage) {
176   std::string VarName = getInstrProfNameVarPrefix();
177   VarName += FuncName;
178 
179   if (!GlobalValue::isLocalLinkage(Linkage))
180     return VarName;
181 
182   // Now fix up illegal chars in local VarName that may upset the assembler.
183   const char *InvalidChars = "-:<>/\"'";
184   size_t found = VarName.find_first_of(InvalidChars);
185   while (found != std::string::npos) {
186     VarName[found] = '_';
187     found = VarName.find_first_of(InvalidChars, found + 1);
188   }
189   return VarName;
190 }
191 
createPGOFuncNameVar(Module & M,GlobalValue::LinkageTypes Linkage,StringRef PGOFuncName)192 GlobalVariable *createPGOFuncNameVar(Module &M,
193                                      GlobalValue::LinkageTypes Linkage,
194                                      StringRef PGOFuncName) {
195 
196   // We generally want to match the function's linkage, but available_externally
197   // and extern_weak both have the wrong semantics, and anything that doesn't
198   // need to link across compilation units doesn't need to be visible at all.
199   if (Linkage == GlobalValue::ExternalWeakLinkage)
200     Linkage = GlobalValue::LinkOnceAnyLinkage;
201   else if (Linkage == GlobalValue::AvailableExternallyLinkage)
202     Linkage = GlobalValue::LinkOnceODRLinkage;
203   else if (Linkage == GlobalValue::InternalLinkage ||
204            Linkage == GlobalValue::ExternalLinkage)
205     Linkage = GlobalValue::PrivateLinkage;
206 
207   auto *Value =
208       ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
209   auto FuncNameVar =
210       new GlobalVariable(M, Value->getType(), true, Linkage, Value,
211                          getPGOFuncNameVarName(PGOFuncName, Linkage));
212 
213   // Hide the symbol so that we correctly get a copy for each executable.
214   if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
215     FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
216 
217   return FuncNameVar;
218 }
219 
createPGOFuncNameVar(Function & F,StringRef PGOFuncName)220 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
221   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
222 }
223 
create(Module & M,bool InLTO)224 void InstrProfSymtab::create(Module &M, bool InLTO) {
225   for (Function &F : M) {
226     // Function may not have a name: like using asm("") to overwrite the name.
227     // Ignore in this case.
228     if (!F.hasName())
229       continue;
230     const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
231     addFuncName(PGOFuncName);
232     MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
233   }
234 
235   finalizeSymtab();
236 }
237 
collectPGOFuncNameStrings(const std::vector<std::string> & NameStrs,bool doCompression,std::string & Result)238 Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs,
239                                 bool doCompression, std::string &Result) {
240   assert(NameStrs.size() && "No name data to emit");
241 
242   uint8_t Header[16], *P = Header;
243   std::string UncompressedNameStrings =
244       join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
245 
246   assert(StringRef(UncompressedNameStrings)
247                  .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
248          "PGO name is invalid (contains separator token)");
249 
250   unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
251   P += EncLen;
252 
253   auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
254     EncLen = encodeULEB128(CompressedLen, P);
255     P += EncLen;
256     char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
257     unsigned HeaderLen = P - &Header[0];
258     Result.append(HeaderStr, HeaderLen);
259     Result += InputStr;
260     return Error::success();
261   };
262 
263   if (!doCompression) {
264     return WriteStringToResult(0, UncompressedNameStrings);
265   }
266 
267   SmallString<128> CompressedNameStrings;
268   zlib::Status Success =
269       zlib::compress(StringRef(UncompressedNameStrings), CompressedNameStrings,
270                      zlib::BestSizeCompression);
271 
272   if (Success != zlib::StatusOK)
273     return make_error<InstrProfError>(instrprof_error::compress_failed);
274 
275   return WriteStringToResult(CompressedNameStrings.size(),
276                              CompressedNameStrings);
277 }
278 
getPGOFuncNameVarInitializer(GlobalVariable * NameVar)279 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
280   auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
281   StringRef NameStr =
282       Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
283   return NameStr;
284 }
285 
collectPGOFuncNameStrings(const std::vector<GlobalVariable * > & NameVars,std::string & Result,bool doCompression)286 Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars,
287                                 std::string &Result, bool doCompression) {
288   std::vector<std::string> NameStrs;
289   for (auto *NameVar : NameVars) {
290     NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar));
291   }
292   return collectPGOFuncNameStrings(
293       NameStrs, zlib::isAvailable() && doCompression, Result);
294 }
295 
readPGOFuncNameStrings(StringRef NameStrings,InstrProfSymtab & Symtab)296 Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
297   const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data());
298   const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() +
299                                                           NameStrings.size());
300   while (P < EndP) {
301     uint32_t N;
302     uint64_t UncompressedSize = decodeULEB128(P, &N);
303     P += N;
304     uint64_t CompressedSize = decodeULEB128(P, &N);
305     P += N;
306     bool isCompressed = (CompressedSize != 0);
307     SmallString<128> UncompressedNameStrings;
308     StringRef NameStrings;
309     if (isCompressed) {
310       StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
311                                       CompressedSize);
312       if (zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
313                            UncompressedSize) != zlib::StatusOK)
314         return make_error<InstrProfError>(instrprof_error::uncompress_failed);
315       P += CompressedSize;
316       NameStrings = StringRef(UncompressedNameStrings.data(),
317                               UncompressedNameStrings.size());
318     } else {
319       NameStrings =
320           StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
321       P += UncompressedSize;
322     }
323     // Now parse the name strings.
324     SmallVector<StringRef, 0> Names;
325     NameStrings.split(Names, getInstrProfNameSeparator());
326     for (StringRef &Name : Names)
327       Symtab.addFuncName(Name);
328 
329     while (P < EndP && *P == 0)
330       P++;
331   }
332   Symtab.finalizeSymtab();
333   return Error::success();
334 }
335 
merge(SoftInstrProfErrors & SIPE,InstrProfValueSiteRecord & Input,uint64_t Weight)336 void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE,
337                                      InstrProfValueSiteRecord &Input,
338                                      uint64_t Weight) {
339   this->sortByTargetValues();
340   Input.sortByTargetValues();
341   auto I = ValueData.begin();
342   auto IE = ValueData.end();
343   for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
344        ++J) {
345     while (I != IE && I->Value < J->Value)
346       ++I;
347     if (I != IE && I->Value == J->Value) {
348       bool Overflowed;
349       I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
350       if (Overflowed)
351         SIPE.addError(instrprof_error::counter_overflow);
352       ++I;
353       continue;
354     }
355     ValueData.insert(I, *J);
356   }
357 }
358 
scale(SoftInstrProfErrors & SIPE,uint64_t Weight)359 void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE,
360                                      uint64_t Weight) {
361   for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
362     bool Overflowed;
363     I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
364     if (Overflowed)
365       SIPE.addError(instrprof_error::counter_overflow);
366   }
367 }
368 
369 // Merge Value Profile data from Src record to this record for ValueKind.
370 // Scale merged value counts by \p Weight.
mergeValueProfData(uint32_t ValueKind,InstrProfRecord & Src,uint64_t Weight)371 void InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
372                                          InstrProfRecord &Src,
373                                          uint64_t Weight) {
374   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
375   uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
376   if (ThisNumValueSites != OtherNumValueSites) {
377     SIPE.addError(instrprof_error::value_site_count_mismatch);
378     return;
379   }
380   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
381       getValueSitesForKind(ValueKind);
382   std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
383       Src.getValueSitesForKind(ValueKind);
384   for (uint32_t I = 0; I < ThisNumValueSites; I++)
385     ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight);
386 }
387 
merge(InstrProfRecord & Other,uint64_t Weight)388 void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) {
389   // If the number of counters doesn't match we either have bad data
390   // or a hash collision.
391   if (Counts.size() != Other.Counts.size()) {
392     SIPE.addError(instrprof_error::count_mismatch);
393     return;
394   }
395 
396   for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
397     bool Overflowed;
398     Counts[I] =
399         SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
400     if (Overflowed)
401       SIPE.addError(instrprof_error::counter_overflow);
402   }
403 
404   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
405     mergeValueProfData(Kind, Other, Weight);
406 }
407 
scaleValueProfData(uint32_t ValueKind,uint64_t Weight)408 void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) {
409   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
410   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
411       getValueSitesForKind(ValueKind);
412   for (uint32_t I = 0; I < ThisNumValueSites; I++)
413     ThisSiteRecords[I].scale(SIPE, Weight);
414 }
415 
scale(uint64_t Weight)416 void InstrProfRecord::scale(uint64_t Weight) {
417   for (auto &Count : this->Counts) {
418     bool Overflowed;
419     Count = SaturatingMultiply(Count, Weight, &Overflowed);
420     if (Overflowed)
421       SIPE.addError(instrprof_error::counter_overflow);
422   }
423   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
424     scaleValueProfData(Kind, Weight);
425 }
426 
427 // Map indirect call target name hash to name string.
remapValue(uint64_t Value,uint32_t ValueKind,ValueMapType * ValueMap)428 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
429                                      ValueMapType *ValueMap) {
430   if (!ValueMap)
431     return Value;
432   switch (ValueKind) {
433   case IPVK_IndirectCallTarget: {
434     auto Result =
435         std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
436                          [](const std::pair<uint64_t, uint64_t> &LHS,
437                             uint64_t RHS) { return LHS.first < RHS; });
438    // Raw function pointer collected by value profiler may be from
439    // external functions that are not instrumented. They won't have
440    // mapping data to be used by the deserializer. Force the value to
441    // be 0 in this case.
442     if (Result != ValueMap->end() && Result->first == Value)
443       Value = (uint64_t)Result->second;
444     else
445       Value = 0;
446     break;
447   }
448   }
449   return Value;
450 }
451 
addValueData(uint32_t ValueKind,uint32_t Site,InstrProfValueData * VData,uint32_t N,ValueMapType * ValueMap)452 void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
453                                    InstrProfValueData *VData, uint32_t N,
454                                    ValueMapType *ValueMap) {
455   for (uint32_t I = 0; I < N; I++) {
456     VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
457   }
458   std::vector<InstrProfValueSiteRecord> &ValueSites =
459       getValueSitesForKind(ValueKind);
460   if (N == 0)
461     ValueSites.emplace_back();
462   else
463     ValueSites.emplace_back(VData, VData + N);
464 }
465 
466 #define INSTR_PROF_COMMON_API_IMPL
467 #include "llvm/ProfileData/InstrProfData.inc"
468 
469 /*!
470  * \brief ValueProfRecordClosure Interface implementation for  InstrProfRecord
471  *  class. These C wrappers are used as adaptors so that C++ code can be
472  *  invoked as callbacks.
473  */
getNumValueKindsInstrProf(const void * Record)474 uint32_t getNumValueKindsInstrProf(const void *Record) {
475   return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
476 }
477 
getNumValueSitesInstrProf(const void * Record,uint32_t VKind)478 uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
479   return reinterpret_cast<const InstrProfRecord *>(Record)
480       ->getNumValueSites(VKind);
481 }
482 
getNumValueDataInstrProf(const void * Record,uint32_t VKind)483 uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
484   return reinterpret_cast<const InstrProfRecord *>(Record)
485       ->getNumValueData(VKind);
486 }
487 
getNumValueDataForSiteInstrProf(const void * R,uint32_t VK,uint32_t S)488 uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
489                                          uint32_t S) {
490   return reinterpret_cast<const InstrProfRecord *>(R)
491       ->getNumValueDataForSite(VK, S);
492 }
493 
getValueForSiteInstrProf(const void * R,InstrProfValueData * Dst,uint32_t K,uint32_t S)494 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
495                               uint32_t K, uint32_t S) {
496   reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
497 }
498 
allocValueProfDataInstrProf(size_t TotalSizeInBytes)499 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
500   ValueProfData *VD =
501       (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
502   memset(VD, 0, TotalSizeInBytes);
503   return VD;
504 }
505 
506 static ValueProfRecordClosure InstrProfRecordClosure = {
507     nullptr,
508     getNumValueKindsInstrProf,
509     getNumValueSitesInstrProf,
510     getNumValueDataInstrProf,
511     getNumValueDataForSiteInstrProf,
512     nullptr,
513     getValueForSiteInstrProf,
514     allocValueProfDataInstrProf};
515 
516 // Wrapper implementation using the closure mechanism.
getSize(const InstrProfRecord & Record)517 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
518   InstrProfRecordClosure.Record = &Record;
519   return getValueProfDataSize(&InstrProfRecordClosure);
520 }
521 
522 // Wrapper implementation using the closure mechanism.
523 std::unique_ptr<ValueProfData>
serializeFrom(const InstrProfRecord & Record)524 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
525   InstrProfRecordClosure.Record = &Record;
526 
527   std::unique_ptr<ValueProfData> VPD(
528       serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
529   return VPD;
530 }
531 
deserializeTo(InstrProfRecord & Record,InstrProfRecord::ValueMapType * VMap)532 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
533                                     InstrProfRecord::ValueMapType *VMap) {
534   Record.reserveSites(Kind, NumValueSites);
535 
536   InstrProfValueData *ValueData = getValueProfRecordValueData(this);
537   for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
538     uint8_t ValueDataCount = this->SiteCountArray[VSite];
539     Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap);
540     ValueData += ValueDataCount;
541   }
542 }
543 
544 // For writing/serializing,  Old is the host endianness, and  New is
545 // byte order intended on disk. For Reading/deserialization, Old
546 // is the on-disk source endianness, and New is the host endianness.
swapBytes(support::endianness Old,support::endianness New)547 void ValueProfRecord::swapBytes(support::endianness Old,
548                                 support::endianness New) {
549   using namespace support;
550   if (Old == New)
551     return;
552 
553   if (getHostEndianness() != Old) {
554     sys::swapByteOrder<uint32_t>(NumValueSites);
555     sys::swapByteOrder<uint32_t>(Kind);
556   }
557   uint32_t ND = getValueProfRecordNumValueData(this);
558   InstrProfValueData *VD = getValueProfRecordValueData(this);
559 
560   // No need to swap byte array: SiteCountArrray.
561   for (uint32_t I = 0; I < ND; I++) {
562     sys::swapByteOrder<uint64_t>(VD[I].Value);
563     sys::swapByteOrder<uint64_t>(VD[I].Count);
564   }
565   if (getHostEndianness() == Old) {
566     sys::swapByteOrder<uint32_t>(NumValueSites);
567     sys::swapByteOrder<uint32_t>(Kind);
568   }
569 }
570 
deserializeTo(InstrProfRecord & Record,InstrProfRecord::ValueMapType * VMap)571 void ValueProfData::deserializeTo(InstrProfRecord &Record,
572                                   InstrProfRecord::ValueMapType *VMap) {
573   if (NumValueKinds == 0)
574     return;
575 
576   ValueProfRecord *VR = getFirstValueProfRecord(this);
577   for (uint32_t K = 0; K < NumValueKinds; K++) {
578     VR->deserializeTo(Record, VMap);
579     VR = getValueProfRecordNext(VR);
580   }
581 }
582 
583 template <class T>
swapToHostOrder(const unsigned char * & D,support::endianness Orig)584 static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
585   using namespace support;
586   if (Orig == little)
587     return endian::readNext<T, little, unaligned>(D);
588   else
589     return endian::readNext<T, big, unaligned>(D);
590 }
591 
allocValueProfData(uint32_t TotalSize)592 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
593   return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
594                                             ValueProfData());
595 }
596 
checkIntegrity()597 Error ValueProfData::checkIntegrity() {
598   if (NumValueKinds > IPVK_Last + 1)
599     return make_error<InstrProfError>(instrprof_error::malformed);
600   // Total size needs to be mulltiple of quadword size.
601   if (TotalSize % sizeof(uint64_t))
602     return make_error<InstrProfError>(instrprof_error::malformed);
603 
604   ValueProfRecord *VR = getFirstValueProfRecord(this);
605   for (uint32_t K = 0; K < this->NumValueKinds; K++) {
606     if (VR->Kind > IPVK_Last)
607       return make_error<InstrProfError>(instrprof_error::malformed);
608     VR = getValueProfRecordNext(VR);
609     if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
610       return make_error<InstrProfError>(instrprof_error::malformed);
611   }
612   return Error::success();
613 }
614 
615 Expected<std::unique_ptr<ValueProfData>>
getValueProfData(const unsigned char * D,const unsigned char * const BufferEnd,support::endianness Endianness)616 ValueProfData::getValueProfData(const unsigned char *D,
617                                 const unsigned char *const BufferEnd,
618                                 support::endianness Endianness) {
619   using namespace support;
620   if (D + sizeof(ValueProfData) > BufferEnd)
621     return make_error<InstrProfError>(instrprof_error::truncated);
622 
623   const unsigned char *Header = D;
624   uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
625   if (D + TotalSize > BufferEnd)
626     return make_error<InstrProfError>(instrprof_error::too_large);
627 
628   std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
629   memcpy(VPD.get(), D, TotalSize);
630   // Byte swap.
631   VPD->swapBytesToHost(Endianness);
632 
633   Error E = VPD->checkIntegrity();
634   if (E)
635     return std::move(E);
636 
637   return std::move(VPD);
638 }
639 
swapBytesToHost(support::endianness Endianness)640 void ValueProfData::swapBytesToHost(support::endianness Endianness) {
641   using namespace support;
642   if (Endianness == getHostEndianness())
643     return;
644 
645   sys::swapByteOrder<uint32_t>(TotalSize);
646   sys::swapByteOrder<uint32_t>(NumValueKinds);
647 
648   ValueProfRecord *VR = getFirstValueProfRecord(this);
649   for (uint32_t K = 0; K < NumValueKinds; K++) {
650     VR->swapBytes(Endianness, getHostEndianness());
651     VR = getValueProfRecordNext(VR);
652   }
653 }
654 
swapBytesFromHost(support::endianness Endianness)655 void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
656   using namespace support;
657   if (Endianness == getHostEndianness())
658     return;
659 
660   ValueProfRecord *VR = getFirstValueProfRecord(this);
661   for (uint32_t K = 0; K < NumValueKinds; K++) {
662     ValueProfRecord *NVR = getValueProfRecordNext(VR);
663     VR->swapBytes(getHostEndianness(), Endianness);
664     VR = NVR;
665   }
666   sys::swapByteOrder<uint32_t>(TotalSize);
667   sys::swapByteOrder<uint32_t>(NumValueKinds);
668 }
669 
annotateValueSite(Module & M,Instruction & Inst,const InstrProfRecord & InstrProfR,InstrProfValueKind ValueKind,uint32_t SiteIdx,uint32_t MaxMDCount)670 void annotateValueSite(Module &M, Instruction &Inst,
671                        const InstrProfRecord &InstrProfR,
672                        InstrProfValueKind ValueKind, uint32_t SiteIdx,
673                        uint32_t MaxMDCount) {
674   uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
675   if (!NV)
676     return;
677 
678   uint64_t Sum = 0;
679   std::unique_ptr<InstrProfValueData[]> VD =
680       InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
681 
682   ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
683   annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
684 }
685 
annotateValueSite(Module & M,Instruction & Inst,ArrayRef<InstrProfValueData> VDs,uint64_t Sum,InstrProfValueKind ValueKind,uint32_t MaxMDCount)686 void annotateValueSite(Module &M, Instruction &Inst,
687                        ArrayRef<InstrProfValueData> VDs,
688                        uint64_t Sum, InstrProfValueKind ValueKind,
689                        uint32_t MaxMDCount) {
690   LLVMContext &Ctx = M.getContext();
691   MDBuilder MDHelper(Ctx);
692   SmallVector<Metadata *, 3> Vals;
693   // Tag
694   Vals.push_back(MDHelper.createString("VP"));
695   // Value Kind
696   Vals.push_back(MDHelper.createConstant(
697       ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
698   // Total Count
699   Vals.push_back(
700       MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
701 
702   // Value Profile Data
703   uint32_t MDCount = MaxMDCount;
704   for (auto &VD : VDs) {
705     Vals.push_back(MDHelper.createConstant(
706         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
707     Vals.push_back(MDHelper.createConstant(
708         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
709     if (--MDCount == 0)
710       break;
711   }
712   Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
713 }
714 
getValueProfDataFromInst(const Instruction & Inst,InstrProfValueKind ValueKind,uint32_t MaxNumValueData,InstrProfValueData ValueData[],uint32_t & ActualNumValueData,uint64_t & TotalC)715 bool getValueProfDataFromInst(const Instruction &Inst,
716                               InstrProfValueKind ValueKind,
717                               uint32_t MaxNumValueData,
718                               InstrProfValueData ValueData[],
719                               uint32_t &ActualNumValueData, uint64_t &TotalC) {
720   MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
721   if (!MD)
722     return false;
723 
724   unsigned NOps = MD->getNumOperands();
725 
726   if (NOps < 5)
727     return false;
728 
729   // Operand 0 is a string tag "VP":
730   MDString *Tag = cast<MDString>(MD->getOperand(0));
731   if (!Tag)
732     return false;
733 
734   if (!Tag->getString().equals("VP"))
735     return false;
736 
737   // Now check kind:
738   ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
739   if (!KindInt)
740     return false;
741   if (KindInt->getZExtValue() != ValueKind)
742     return false;
743 
744   // Get total count
745   ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
746   if (!TotalCInt)
747     return false;
748   TotalC = TotalCInt->getZExtValue();
749 
750   ActualNumValueData = 0;
751 
752   for (unsigned I = 3; I < NOps; I += 2) {
753     if (ActualNumValueData >= MaxNumValueData)
754       break;
755     ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
756     ConstantInt *Count =
757         mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
758     if (!Value || !Count)
759       return false;
760     ValueData[ActualNumValueData].Value = Value->getZExtValue();
761     ValueData[ActualNumValueData].Count = Count->getZExtValue();
762     ActualNumValueData++;
763   }
764   return true;
765 }
766 
getPGOFuncNameMetadata(const Function & F)767 MDNode *getPGOFuncNameMetadata(const Function &F) {
768   return F.getMetadata(getPGOFuncNameMetadataName());
769 }
770 
createPGOFuncNameMetadata(Function & F,StringRef PGOFuncName)771 void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
772   // Only for internal linkage functions.
773   if (PGOFuncName == F.getName())
774       return;
775   // Don't create duplicated meta-data.
776   if (getPGOFuncNameMetadata(F))
777     return;
778   LLVMContext &C = F.getContext();
779   MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
780   F.setMetadata(getPGOFuncNameMetadataName(), N);
781 }
782 
783 } // end namespace llvm
784