SledKind Kind;
bool AlwaysInstrument;
const class Function *Fn;
+
+ void emit(int, MCStreamer *, const MCSymbol *) const;
};
// All the sleds to be emitted.
// Helper function to record a given XRay sled.
void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind);
+ /// Emit a table with all XRay instrumentation points.
+ void emitXRayTable();
+
//===------------------------------------------------------------------===//
// MachineFunctionPass Implementation.
//===------------------------------------------------------------------===//
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCSectionMachO.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbolELF.h"
#include "llvm/MC/MCValue.h"
void AsmPrinterHandler::markFunctionEnd() {}
+// In the binary's "xray_instr_map" section, an array of these function entries
+// describes each instrumentation point. When XRay patches your code, the index
+// into this table will be given to your handler as a patch point identifier.
+void AsmPrinter::XRayFunctionEntry::emit(int Bytes, MCStreamer *Out,
+ const MCSymbol *CurrentFnSym) const {
+ Out->EmitSymbolValue(Sled, Bytes);
+ Out->EmitSymbolValue(CurrentFnSym, Bytes);
+ auto Kind8 = static_cast<uint8_t>(Kind);
+ Out->EmitBytes(StringRef(reinterpret_cast<const char *>(&Kind8), 1));
+ Out->EmitBytes(
+ StringRef(reinterpret_cast<const char *>(&AlwaysInstrument), 1));
+ Out->EmitZeros(2 * Bytes - 2); // Pad the previous two entries
+}
+
+void AsmPrinter::emitXRayTable() {
+ if (Sleds.empty())
+ return;
+
+ auto PrevSection = OutStreamer->getCurrentSectionOnly();
+ auto Fn = MF->getFunction();
+ MCSection *Section = nullptr;
+ if (MF->getSubtarget().getTargetTriple().isOSBinFormatELF()) {
+ if (Fn->hasComdat()) {
+ Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
+ ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
+ Fn->getComdat()->getName());
+ } else {
+ Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
+ ELF::SHF_ALLOC);
+ }
+ } else if (MF->getSubtarget().getTargetTriple().isOSBinFormatMachO()) {
+ Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
+ SectionKind::getReadOnlyWithRel());
+ } else {
+ llvm_unreachable("Unsupported target");
+ }
+
+ // Before we switch over, we force a reference to a label inside the
+ // xray_instr_map section. Since this function is always called just
+ // before the function's end, we assume that this is happening after
+ // the last return instruction.
+
+ auto WordSizeBytes = TM.getPointerSize();
+ MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true);
+ OutStreamer->EmitCodeAlignment(16);
+ OutStreamer->EmitSymbolValue(Tmp, WordSizeBytes, false);
+ OutStreamer->SwitchSection(Section);
+ OutStreamer->EmitLabel(Tmp);
+ for (const auto &Sled : Sleds)
+ Sled.emit(WordSizeBytes, OutStreamer.get(), CurrentFnSym);
+
+ OutStreamer->SwitchSection(PrevSection);
+ Sleds.clear();
+}
+
void AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
SledKind Kind) {
auto Fn = MI.getParent()->getParent()->getFunction();
void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
- void EmitXRayTable();
void EmitSled(const MachineInstr &MI, SledKind Kind);
/// \brief tblgen'erated driver function for lowering simple MI->MC
AArch64FI = F.getInfo<AArch64FunctionInfo>();
STI = static_cast<const AArch64Subtarget*>(&F.getSubtarget());
bool Result = AsmPrinter::runOnMachineFunction(F);
- EmitXRayTable();
+ emitXRayTable();
return Result;
}
EmitSled(MI, SledKind::TAIL_CALL);
}
-void AArch64AsmPrinter::EmitXRayTable()
-{
- //TODO: merge the logic for ELF XRay sleds at a higher level, so to avoid
- // code duplication as it is now for x86_64, ARM32 and AArch64.
- if (Sleds.empty())
- return;
-
- auto PrevSection = OutStreamer->getCurrentSectionOnly();
- auto Fn = MF->getFunction();
- MCSection *Section;
-
- if (STI->isTargetELF()) {
- if (Fn->hasComdat())
- Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
- Fn->getComdat()->getName());
- else
- Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC);
- } else if (STI->isTargetMachO()) {
- Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
- SectionKind::getReadOnlyWithRel());
- } else {
- llvm_unreachable("Unsupported target");
- }
-
- // Before we switch over, we force a reference to a label inside the
- // xray_instr_map section. Since EmitXRayTable() is always called just
- // before the function's end, we assume that this is happening after the
- // last return instruction.
- //
- // We then align the reference to 16 byte boundaries, which we determined
- // experimentally to be beneficial to avoid causing decoder stalls.
- MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true);
- OutStreamer->EmitCodeAlignment(16);
- OutStreamer->EmitSymbolValue(Tmp, 8, false);
- OutStreamer->SwitchSection(Section);
- OutStreamer->EmitLabel(Tmp);
- for (const auto &Sled : Sleds) {
- OutStreamer->EmitSymbolValue(Sled.Sled, 8);
- OutStreamer->EmitSymbolValue(CurrentFnSym, 8);
- auto Kind = static_cast<uint8_t>(Sled.Kind);
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Kind), 1));
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
- OutStreamer->EmitZeros(14);
- }
- OutStreamer->SwitchSection(PrevSection);
-
- Sleds.clear();
-}
-
void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
{
static const int8_t NoopsInSledCount = 7;
// Emit the rest of the function body.
EmitFunctionBody();
- // Emit the XRay table for this function.
- EmitXRayTable();
-
// If we need V4T thumb mode Register Indirect Jump pads, emit them.
// These are created per function, rather than per TU, since it's
// relatively easy to exceed the thumb branch range within a TU.
void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
- // Helper function that emits the XRay sleds we've collected for a particular
- // function.
- void EmitXRayTable();
private:
void EmitSled(const MachineInstr &MI, SledKind Kind);
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCSymbolELF.h"
-#include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCSectionMachO.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCStreamer.h"
using namespace llvm;
{
EmitSled(MI, SledKind::TAIL_CALL);
}
-
-void ARMAsmPrinter::EmitXRayTable()
-{
- if (Sleds.empty())
- return;
-
- MCSection *Section = nullptr;
- if (Subtarget->isTargetELF()) {
- Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC | ELF::SHF_GROUP |
- ELF::SHF_MERGE,
- 0, CurrentFnSym->getName());
- } else if (Subtarget->isTargetMachO()) {
- Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
- SectionKind::getReadOnlyWithRel());
- } else {
- llvm_unreachable("Unsupported target");
- }
-
- auto PrevSection = OutStreamer->getCurrentSectionOnly();
- OutStreamer->SwitchSection(Section);
- for (const auto &Sled : Sleds) {
- OutStreamer->EmitSymbolValue(Sled.Sled, 4);
- OutStreamer->EmitSymbolValue(CurrentFnSym, 4);
- auto Kind = static_cast<uint8_t>(Sled.Kind);
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Kind), 1));
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
- OutStreamer->EmitZeros(6);
- }
- OutStreamer->SwitchSection(PrevSection);
-
- Sleds.clear();
-}
EmitFunctionBody();
// Emit the XRay table for this function.
- EmitXRayTable();
+ emitXRayTable();
// We didn't modify anything.
return false;
OutStreamer->EmitInstruction(TC, getSubtargetInfo());
}
-void X86AsmPrinter::EmitXRayTable() {
- if (Sleds.empty())
- return;
-
- auto PrevSection = OutStreamer->getCurrentSectionOnly();
- auto Fn = MF->getFunction();
- MCSection *Section = nullptr;
- if (Subtarget->isTargetELF()) {
- if (Fn->hasComdat()) {
- Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
- Fn->getComdat()->getName());
- } else {
- Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
- ELF::SHF_ALLOC);
- }
- } else if (Subtarget->isTargetMachO()) {
- Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
- SectionKind::getReadOnlyWithRel());
- } else {
- llvm_unreachable("Unsupported target");
- }
-
- // Before we switch over, we force a reference to a label inside the
- // xray_instr_map section. Since EmitXRayTable() is always called just
- // before the function's end, we assume that this is happening after the
- // last return instruction.
- //
- // We then align the reference to 16 byte boundaries, which we determined
- // experimentally to be beneficial to avoid causing decoder stalls.
- MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true);
- OutStreamer->EmitCodeAlignment(16);
- OutStreamer->EmitSymbolValue(Tmp, 8, false);
- OutStreamer->SwitchSection(Section);
- OutStreamer->EmitLabel(Tmp);
- for (const auto &Sled : Sleds) {
- OutStreamer->EmitSymbolValue(Sled.Sled, 8);
- OutStreamer->EmitSymbolValue(CurrentFnSym, 8);
- auto Kind = static_cast<uint8_t>(Sled.Kind);
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Kind), 1));
- OutStreamer->EmitBytes(
- StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
- OutStreamer->EmitZeros(14);
- }
- OutStreamer->SwitchSection(PrevSection);
-
- Sleds.clear();
-}
-
// Returns instruction preceding MBBI in MachineFunction.
// If MBBI is the first instruction of the first basic block, returns null.
static MachineBasicBlock::const_iterator