-//=-- InstrProf.cpp - Instrumented profiling format support -----------------=//
+//===- InstrProf.cpp - Instrumented profiling format support --------------===//
//
// The LLVM Compiler Infrastructure
//
//===----------------------------------------------------------------------===//
#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
#include "llvm/Support/Compression.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/SwapByteOrder.h"
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
using namespace llvm;
static cl::opt<bool> StaticFuncFullModulePrefix(
- "static-func-full-module-prefix", cl::init(true),
+ "static-func-full-module-prefix", cl::init(true), cl::Hidden,
cl::desc("Use full module build paths in the profile counter names for "
"static functions."));
// of levels to strip. A value larger than the number of directories in the
// source file will strip all the directory names and only leave the basename.
//
-// Note current Thin-LTO module importing for the indirect-calls assumes
+// Note current ThinLTO module importing for the indirect-calls assumes
// the source directory name not being stripped. A non-zero option value here
// can potentially prevent some inter-module indirect-call-promotions.
static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
- "static-func-strip-dirname-prefix", cl::init(0),
+ "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
cl::desc("Strip specified level of directory name from source path in "
"the profile counter name for static functions."));
-namespace {
-std::string getInstrProfErrString(instrprof_error Err) {
+static std::string getInstrProfErrString(instrprof_error Err) {
switch (Err) {
case instrprof_error::success:
return "Success";
return "Failed to uncompress data (zlib)";
case instrprof_error::empty_raw_profile:
return "Empty raw profile file";
+ case instrprof_error::zlib_unavailable:
+ return "Profile uses zlib compression but the profile reader was built without zlib support";
}
llvm_unreachable("A value of instrprof_error has no message.");
}
+namespace {
+
// FIXME: This class is only here to support the transition to llvm::Error. It
// will be removed once this transition is complete. Clients should prefer to
// deal with the Error value directly, rather than converting to error_code.
class InstrProfErrorCategoryType : public std::error_category {
const char *name() const noexcept override { return "llvm.instrprof"; }
+
std::string message(int IE) const override {
return getInstrProfErrString(static_cast<instrprof_error>(IE));
}
};
+
} // end anonymous namespace
static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
return *ErrorCategory;
}
+namespace {
+
+const char *InstrProfSectNameCommon[] = {
+#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
+ SectNameCommon,
+#include "llvm/ProfileData/InstrProfData.inc"
+};
+
+const char *InstrProfSectNameCoff[] = {
+#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
+ SectNameCoff,
+#include "llvm/ProfileData/InstrProfData.inc"
+};
+
+const char *InstrProfSectNamePrefix[] = {
+#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
+ Prefix,
+#include "llvm/ProfileData/InstrProfData.inc"
+};
+
+} // namespace
+
namespace llvm {
+std::string getInstrProfSectionName(InstrProfSectKind IPSK,
+ Triple::ObjectFormatType OF,
+ bool AddSegmentInfo) {
+ std::string SectName;
+
+ if (OF == Triple::MachO && AddSegmentInfo)
+ SectName = InstrProfSectNamePrefix[IPSK];
+
+ if (OF == Triple::COFF)
+ SectName += InstrProfSectNameCoff[IPSK];
+ else
+ SectName += InstrProfSectNameCommon[IPSK];
+
+ if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
+ SectName += ",regular,live_support";
+
+ return SectName;
+}
+
void SoftInstrProfErrors::addError(instrprof_error IE) {
if (IE == instrprof_error::success)
return;
GlobalVariable *createPGOFuncNameVar(Module &M,
GlobalValue::LinkageTypes Linkage,
StringRef PGOFuncName) {
-
// We generally want to match the function's linkage, but available_externally
// and extern_weak both have the wrong semantics, and anything that doesn't
// need to link across compilation units doesn't need to be visible at all.
return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
}
-void InstrProfSymtab::create(Module &M, bool InLTO) {
+Error InstrProfSymtab::create(Module &M, bool InLTO) {
for (Function &F : M) {
// Function may not have a name: like using asm("") to overwrite the name.
// Ignore in this case.
if (!F.hasName())
continue;
const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
- addFuncName(PGOFuncName);
+ if (Error E = addFuncName(PGOFuncName))
+ return E;
MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
+ // In ThinLTO, local function may have been promoted to global and have
+ // suffix added to the function name. We need to add the stripped function
+ // name to the symbol table so that we can find a match from profile.
+ if (InLTO) {
+ auto pos = PGOFuncName.find('.');
+ if (pos != std::string::npos) {
+ const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
+ if (Error E = addFuncName(OtherFuncName))
+ return E;
+ MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
+ }
+ }
}
-
+ Sorted = false;
finalizeSymtab();
+ return Error::success();
}
-Error collectPGOFuncNameStrings(const std::vector<std::string> &NameStrs,
+uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
+ finalizeSymtab();
+ auto Result =
+ std::lower_bound(AddrToMD5Map.begin(), AddrToMD5Map.end(), Address,
+ [](const std::pair<uint64_t, uint64_t> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
+ // Raw function pointer collected by value profiler may be from
+ // external functions that are not instrumented. They won't have
+ // mapping data to be used by the deserializer. Force the value to
+ // be 0 in this case.
+ if (Result != AddrToMD5Map.end() && Result->first == Address)
+ return (uint64_t)Result->second;
+ return 0;
+}
+
+Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
bool doCompression, std::string &Result) {
- assert(NameStrs.size() && "No name data to emit");
+ assert(!NameStrs.empty() && "No name data to emit");
uint8_t Header[16], *P = Header;
std::string UncompressedNameStrings =
return NameStr;
}
-Error collectPGOFuncNameStrings(const std::vector<GlobalVariable *> &NameVars,
+Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
std::string &Result, bool doCompression) {
std::vector<std::string> NameStrs;
for (auto *NameVar : NameVars) {
SmallString<128> UncompressedNameStrings;
StringRef NameStrings;
if (isCompressed) {
+ if (!llvm::zlib::isAvailable())
+ return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
+
StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
CompressedSize);
if (Error E =
SmallVector<StringRef, 0> Names;
NameStrings.split(Names, getInstrProfNameSeparator());
for (StringRef &Name : Names)
- Symtab.addFuncName(Name);
+ if (Error E = Symtab.addFuncName(Name))
+ return E;
while (P < EndP && *P == 0)
P++;
}
- Symtab.finalizeSymtab();
return Error::success();
}
-void InstrProfValueSiteRecord::merge(SoftInstrProfErrors &SIPE,
- InstrProfValueSiteRecord &Input,
- uint64_t Weight) {
+void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
+ uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
this->sortByTargetValues();
Input.sortByTargetValues();
auto I = ValueData.begin();
bool Overflowed;
I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
if (Overflowed)
- SIPE.addError(instrprof_error::counter_overflow);
+ Warn(instrprof_error::counter_overflow);
++I;
continue;
}
}
}
-void InstrProfValueSiteRecord::scale(SoftInstrProfErrors &SIPE,
- uint64_t Weight) {
+void InstrProfValueSiteRecord::scale(uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
bool Overflowed;
I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
if (Overflowed)
- SIPE.addError(instrprof_error::counter_overflow);
+ Warn(instrprof_error::counter_overflow);
}
}
// Merge Value Profile data from Src record to this record for ValueKind.
// Scale merged value counts by \p Weight.
-void InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
- InstrProfRecord &Src,
- uint64_t Weight) {
+void InstrProfRecord::mergeValueProfData(
+ uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
if (ThisNumValueSites != OtherNumValueSites) {
- SIPE.addError(instrprof_error::value_site_count_mismatch);
+ Warn(instrprof_error::value_site_count_mismatch);
return;
}
+ if (!ThisNumValueSites)
+ return;
std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
- getValueSitesForKind(ValueKind);
- std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
+ getOrCreateValueSitesForKind(ValueKind);
+ MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
Src.getValueSitesForKind(ValueKind);
for (uint32_t I = 0; I < ThisNumValueSites; I++)
- ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight);
+ ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
}
-void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) {
+void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
// If the number of counters doesn't match we either have bad data
// or a hash collision.
if (Counts.size() != Other.Counts.size()) {
- SIPE.addError(instrprof_error::count_mismatch);
+ Warn(instrprof_error::count_mismatch);
return;
}
Counts[I] =
SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
if (Overflowed)
- SIPE.addError(instrprof_error::counter_overflow);
+ Warn(instrprof_error::counter_overflow);
}
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
- mergeValueProfData(Kind, Other, Weight);
+ mergeValueProfData(Kind, Other, Weight, Warn);
}
-void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) {
- uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
- std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
- getValueSitesForKind(ValueKind);
- for (uint32_t I = 0; I < ThisNumValueSites; I++)
- ThisSiteRecords[I].scale(SIPE, Weight);
+void InstrProfRecord::scaleValueProfData(
+ uint32_t ValueKind, uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
+ for (auto &R : getValueSitesForKind(ValueKind))
+ R.scale(Weight, Warn);
}
-void InstrProfRecord::scale(uint64_t Weight) {
+void InstrProfRecord::scale(uint64_t Weight,
+ function_ref<void(instrprof_error)> Warn) {
for (auto &Count : this->Counts) {
bool Overflowed;
Count = SaturatingMultiply(Count, Weight, &Overflowed);
if (Overflowed)
- SIPE.addError(instrprof_error::counter_overflow);
+ Warn(instrprof_error::counter_overflow);
}
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
- scaleValueProfData(Kind, Weight);
+ scaleValueProfData(Kind, Weight, Warn);
}
// Map indirect call target name hash to name string.
uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
- ValueMapType *ValueMap) {
- if (!ValueMap)
+ InstrProfSymtab *SymTab) {
+ if (!SymTab)
return Value;
- switch (ValueKind) {
- case IPVK_IndirectCallTarget: {
- auto Result =
- std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
- [](const std::pair<uint64_t, uint64_t> &LHS,
- uint64_t RHS) { return LHS.first < RHS; });
- // Raw function pointer collected by value profiler may be from
- // external functions that are not instrumented. They won't have
- // mapping data to be used by the deserializer. Force the value to
- // be 0 in this case.
- if (Result != ValueMap->end() && Result->first == Value)
- Value = (uint64_t)Result->second;
- else
- Value = 0;
- break;
- }
- }
+
+ if (ValueKind == IPVK_IndirectCallTarget)
+ return SymTab->getFunctionHashFromAddress(Value);
+
return Value;
}
void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
InstrProfValueData *VData, uint32_t N,
- ValueMapType *ValueMap) {
+ InstrProfSymtab *ValueMap) {
for (uint32_t I = 0; I < N; I++) {
VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
}
std::vector<InstrProfValueSiteRecord> &ValueSites =
- getValueSitesForKind(ValueKind);
+ getOrCreateValueSitesForKind(ValueKind);
if (N == 0)
ValueSites.emplace_back();
else
#include "llvm/ProfileData/InstrProfData.inc"
/*!
- * \brief ValueProfRecordClosure Interface implementation for InstrProfRecord
+ * ValueProfRecordClosure Interface implementation for InstrProfRecord
* class. These C wrappers are used as adaptors so that C++ code can be
* invoked as callbacks.
*/
// Wrapper implementation using the closure mechanism.
uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
- InstrProfRecordClosure.Record = &Record;
- return getValueProfDataSize(&InstrProfRecordClosure);
+ auto Closure = InstrProfRecordClosure;
+ Closure.Record = &Record;
+ return getValueProfDataSize(&Closure);
}
// Wrapper implementation using the closure mechanism.
}
void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
- InstrProfRecord::ValueMapType *VMap) {
+ InstrProfSymtab *SymTab) {
Record.reserveSites(Kind, NumValueSites);
InstrProfValueData *ValueData = getValueProfRecordValueData(this);
for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
uint8_t ValueDataCount = this->SiteCountArray[VSite];
- Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap);
+ Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
ValueData += ValueDataCount;
}
}
void ValueProfRecord::swapBytes(support::endianness Old,
support::endianness New) {
using namespace support;
+
if (Old == New)
return;
}
void ValueProfData::deserializeTo(InstrProfRecord &Record,
- InstrProfRecord::ValueMapType *VMap) {
+ InstrProfSymtab *SymTab) {
if (NumValueKinds == 0)
return;
ValueProfRecord *VR = getFirstValueProfRecord(this);
for (uint32_t K = 0; K < NumValueKinds; K++) {
- VR->deserializeTo(Record, VMap);
+ VR->deserializeTo(Record, SymTab);
VR = getValueProfRecordNext(VR);
}
}
template <class T>
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
using namespace support;
+
if (Orig == little)
return endian::readNext<T, little, unaligned>(D);
else
const unsigned char *const BufferEnd,
support::endianness Endianness) {
using namespace support;
+
if (D + sizeof(ValueProfData) > BufferEnd)
return make_error<InstrProfError>(instrprof_error::truncated);
void ValueProfData::swapBytesToHost(support::endianness Endianness) {
using namespace support;
+
if (Endianness == getHostEndianness())
return;
void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
using namespace support;
+
if (Endianness == getHostEndianness())
return;
if (F.hasComdat())
return true;
- Triple TT(M.getTargetTriple());
- if (!TT.isOSBinFormatELF() && !TT.isOSBinFormatWasm())
+ if (!Triple(M.getTargetTriple()).supportsCOMDAT())
return false;
// See createPGOFuncNameVar for more details. To avoid link errors, profile
}
return true;
}
+
+// Parse the value profile options.
+void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
+ int64_t &RangeLast) {
+ static const int64_t DefaultMemOPSizeRangeStart = 0;
+ static const int64_t DefaultMemOPSizeRangeLast = 8;
+ RangeStart = DefaultMemOPSizeRangeStart;
+ RangeLast = DefaultMemOPSizeRangeLast;
+
+ if (!MemOPSizeRange.empty()) {
+ auto Pos = MemOPSizeRange.find(':');
+ if (Pos != std::string::npos) {
+ if (Pos > 0)
+ MemOPSizeRange.substr(0, Pos).getAsInteger(10, RangeStart);
+ if (Pos < MemOPSizeRange.size() - 1)
+ MemOPSizeRange.substr(Pos + 1).getAsInteger(10, RangeLast);
+ } else
+ MemOPSizeRange.getAsInteger(10, RangeLast);
+ }
+ assert(RangeLast >= RangeStart);
+}
+
} // end namespace llvm
OSDN Git Service
