// Collect timing for just the portion that constructs the live
// range intervals based on the end-of-live-range computation, for a
// finer breakdown of the cost.
+ TimerMarker T1(TimerStack::TT_liveRange, this);
// Make a final pass over instructions to delete dead instructions
// and build each Variable's live range.
- TimerMarker T1(TimerStack::TT_liveRange, this);
for (CfgNode *Node : Nodes)
Node->livenessPostprocess(Mode, getLiveness());
if (Mode == Liveness_Intervals) {
if (Var->getWeight().isInf())
Var->setLiveRangeInfiniteWeight();
}
- dump();
}
}
bool Valid = true;
Ostream &Str = Ctx->getStrDump();
for (CfgNode *Node : Nodes) {
+ Inst *FirstInst = NULL;
for (Inst *Inst : Node->getInsts()) {
if (Inst->isDeleted())
continue;
if (llvm::isa<InstFakeKill>(Inst))
continue;
+ if (FirstInst == NULL)
+ FirstInst = Inst;
InstNumberT InstNumber = Inst->getNumber();
- Variable *Dest = Inst->getDest();
- if (Dest) {
- // TODO: This instruction should actually begin Dest's live
- // range, so we could probably test that this instruction is
- // the beginning of some segment of Dest's live range. But
- // this wouldn't work with non-SSA temporaries during
- // lowering.
- if (!Dest->getLiveRange().containsValue(InstNumber)) {
- Valid = false;
- Str << "Liveness error: inst " << Inst->getNumber() << " dest ";
- Dest->dump(this);
- Str << " live range " << Dest->getLiveRange() << "\n";
+ if (Variable *Dest = Inst->getDest()) {
+ if (!Dest->getIgnoreLiveness()) {
+ bool Invalid = false;
+ const bool IsDest = true;
+ if (!Dest->getLiveRange().containsValue(InstNumber, IsDest))
+ Invalid = true;
+ // Check that this instruction actually *begins* Dest's live
+ // range, by checking that Dest is not live in the previous
+ // instruction. As a special exception, we don't check this
+ // for the first instruction of the block, because a Phi
+ // temporary may be live at the end of the previous block,
+ // and if it is also assigned in the first instruction of
+ // this block, the adjacent live ranges get merged.
+ if (Inst != FirstInst && !Inst->isDestNonKillable() &&
+ Dest->getLiveRange().containsValue(InstNumber - 1, IsDest))
+ Invalid = true;
+ if (Invalid) {
+ Valid = false;
+ Str << "Liveness error: inst " << Inst->getNumber() << " dest ";
+ Dest->dump(this);
+ Str << " live range " << Dest->getLiveRange() << "\n";
+ }
}
}
for (SizeT I = 0; I < Inst->getSrcSize(); ++I) {
SizeT NumVars = Src->getNumVars();
for (SizeT J = 0; J < NumVars; ++J) {
const Variable *Var = Src->getVar(J);
- if (!Var->getLiveRange().containsValue(InstNumber)) {
+ const bool IsDest = false;
+ if (!Var->getIgnoreLiveness() &&
+ !Var->getLiveRange().containsValue(InstNumber, IsDest)) {
Valid = false;
Str << "Liveness error: inst " << Inst->getNumber() << " var ";
Var->dump(this);
// Manage instruction numbering.
InstNumberT newInstNumber() { return NextInstNumber++; }
+ InstNumberT getNextInstNumber() const { return NextInstNumber; }
// Manage Variables.
// Create a new Variable with a particular type and an optional
namespace Ice {
CfgNode::CfgNode(Cfg *Func, SizeT LabelNumber, IceString Name)
- : Func(Func), Number(LabelNumber), Name(Name), HasReturn(false) {}
+ : Func(Func), Number(LabelNumber), Name(Name), HasReturn(false),
+ InstCountEstimate(0) {}
// Returns the name the node was created with. If no name was given,
// it synthesizes a (hopefully) unique name.
// instruction list. Validates that all Phis are added before all
// regular instructions.
void CfgNode::appendInst(Inst *Inst) {
+ ++InstCountEstimate;
if (InstPhi *Phi = llvm::dyn_cast<InstPhi>(Inst)) {
if (!Insts.empty()) {
Func->setError("Phi instruction added to the middle of a block");
// instruction numbers in a block, from lowest to highest, must not
// overlap with the range of any other block.
void CfgNode::renumberInstructions() {
+ InstNumberT FirstNumber = Func->getNextInstNumber();
for (InstPhi *I : Phis)
I->renumber(Func);
for (Inst *I : Insts)
I->renumber(Func);
+ InstCountEstimate = Func->getNextInstNumber() - FirstNumber;
}
// When a node is created, the OutEdges are immediately knows, but the
void CfgNode::livenessLightweight() {
SizeT NumVars = Func->getNumVariables();
- llvm::BitVector Live(NumVars);
+ LivenessBV Live(NumVars);
// Process regular instructions in reverse order.
// TODO(stichnot): Use llvm::make_range with LLVM 3.5.
for (auto I = Insts.rbegin(), E = Insts.rend(); I != E; ++I) {
// again.)
bool CfgNode::liveness(Liveness *Liveness) {
SizeT NumVars = Liveness->getNumVarsInNode(this);
- llvm::BitVector Live(NumVars);
+ LivenessBV Live(NumVars);
+ LiveBeginEndMap *LiveBegin = NULL;
+ LiveBeginEndMap *LiveEnd = NULL;
// Mark the beginning and ending of each variable's live range
// with the sentinel instruction number 0.
- std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(this);
- std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(this);
- InstNumberT Sentinel = Inst::NumberSentinel;
- LiveBegin.assign(NumVars, Sentinel);
- LiveEnd.assign(NumVars, Sentinel);
+ if (Liveness->getMode() == Liveness_Intervals) {
+ LiveBegin = Liveness->getLiveBegin(this);
+ LiveEnd = Liveness->getLiveEnd(this);
+ LiveBegin->clear();
+ LiveEnd->clear();
+ // Guess that the number of live ranges beginning is roughly the
+ // number of instructions, and same for live ranges ending.
+ LiveBegin->reserve(getInstCountEstimate());
+ LiveEnd->reserve(getInstCountEstimate());
+ }
// Initialize Live to be the union of all successors' LiveIn.
for (CfgNode *Succ : OutEdges) {
Live |= Liveness->getLiveIn(Succ);
for (auto I = Insts.rbegin(), E = Insts.rend(); I != E; ++I) {
if ((*I)->isDeleted())
continue;
- (*I)->liveness((*I)->getNumber(), Live, Liveness, this);
+ (*I)->liveness((*I)->getNumber(), Live, Liveness, LiveBegin, LiveEnd);
}
// Process phis in forward order so that we can override the
// instruction number to be that of the earliest phi instruction in
continue;
if (FirstPhiNumber == Inst::NumberSentinel)
FirstPhiNumber = I->getNumber();
- I->liveness(FirstPhiNumber, Live, Liveness, this);
+ I->liveness(FirstPhiNumber, Live, Liveness, LiveBegin, LiveEnd);
}
// When using the sparse representation, after traversing the
// by shrinking the Live vector and then testing it against the
// pre-shrunk version. (The shrinking is required, but the
// validation is not.)
- llvm::BitVector LiveOrig = Live;
+ LivenessBV LiveOrig = Live;
Live.resize(Liveness->getNumGlobalVars());
// Non-global arguments in the entry node are allowed to be live on
// entry.
}
bool Changed = false;
- llvm::BitVector &LiveIn = Liveness->getLiveIn(this);
+ LivenessBV &LiveIn = Liveness->getLiveIn(this);
// Add in current LiveIn
Live |= LiveIn;
// Check result, set LiveIn=Live
return Changed;
}
+#ifndef NDEBUG
+namespace {
+
+bool comparePair(const LiveBeginEndMapEntry &A, const LiveBeginEndMapEntry &B) {
+ return A.first == B.first;
+}
+
+} // end of anonymous namespace
+#endif // NDEBUG
+
// Now that basic liveness is complete, remove dead instructions that
// were tentatively marked as dead, and compute actual live ranges.
// It is assumed that within a single basic block, a live range begins
TimerMarker T1(TimerStack::TT_liveRangeCtor, Func);
SizeT NumVars = Liveness->getNumVarsInNode(this);
- SizeT NumGlobals = Liveness->getNumGlobalVars();
- llvm::BitVector &LiveIn = Liveness->getLiveIn(this);
- llvm::BitVector &LiveOut = Liveness->getLiveOut(this);
- std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(this);
- std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(this);
- for (SizeT i = 0; i < NumVars; ++i) {
- // Deal with the case where the variable is both live-in and
- // live-out, but LiveEnd comes before LiveBegin. In this case, we
- // need to add two segments to the live range because there is a
- // hole in the middle. This would typically happen as a result of
- // phi lowering in the presence of loopback edges.
- bool IsGlobal = (i < NumGlobals);
- if (IsGlobal && LiveIn[i] && LiveOut[i] && LiveBegin[i] > LiveEnd[i]) {
- Variable *Var = Liveness->getVariable(i, this);
- Liveness->addLiveRange(Var, FirstInstNum, LiveEnd[i], 1);
- Liveness->addLiveRange(Var, LiveBegin[i], LastInstNum + 1, 1);
- continue;
+ LivenessBV &LiveIn = Liveness->getLiveIn(this);
+ LivenessBV &LiveOut = Liveness->getLiveOut(this);
+ LiveBeginEndMap &MapBegin = *Liveness->getLiveBegin(this);
+ LiveBeginEndMap &MapEnd = *Liveness->getLiveEnd(this);
+ std::sort(MapBegin.begin(), MapBegin.end());
+ std::sort(MapEnd.begin(), MapEnd.end());
+ // Verify there are no duplicates.
+ assert(std::adjacent_find(MapBegin.begin(), MapBegin.end(), comparePair) ==
+ MapBegin.end());
+ assert(std::adjacent_find(MapEnd.begin(), MapEnd.end(), comparePair) ==
+ MapEnd.end());
+
+ LivenessBV LiveInAndOut = LiveIn;
+ LiveInAndOut &= LiveOut;
+
+ // Iterate in parallel across the sorted MapBegin[] and MapEnd[].
+ auto IBB = MapBegin.begin(), IEB = MapEnd.begin();
+ auto IBE = MapBegin.end(), IEE = MapEnd.end();
+ while (IBB != IBE || IEB != IEE) {
+ SizeT i1 = IBB == IBE ? NumVars : IBB->first;
+ SizeT i2 = IEB == IEE ? NumVars : IEB->first;
+ SizeT i = std::min(i1, i2);
+ // i1 is the Variable number of the next MapBegin entry, and i2 is
+ // the Variable number of the next MapEnd entry. If i1==i2, then
+ // the Variable's live range begins and ends in this block. If
+ // i1<i2, then i1's live range begins at instruction IBB->second
+ // and extends through the end of the block. If i1>i2, then i2's
+ // live range begins at the first instruction of the block and
+ // ends at IEB->second. In any case, we choose the lesser of i1
+ // and i2 and proceed accordingly.
+ InstNumberT LB = i == i1 ? IBB->second : FirstInstNum;
+ InstNumberT LE = i == i2 ? IEB->second : LastInstNum + 1;
+
+ Variable *Var = Liveness->getVariable(i, this);
+ if (!Var->getIgnoreLiveness()) {
+ if (LB > LE) {
+ Liveness->addLiveRange(Var, FirstInstNum, LE, 1);
+ Liveness->addLiveRange(Var, LB, LastInstNum + 1, 1);
+ // Assert that Var is a global variable by checking that its
+ // liveness index is less than the number of globals. This
+ // ensures that the LiveInAndOut[] access is valid.
+ assert(i < Liveness->getNumGlobalVars());
+ LiveInAndOut[i] = false;
+ } else {
+ Liveness->addLiveRange(Var, LB, LE, 1);
+ }
}
- InstNumberT Begin = (IsGlobal && LiveIn[i]) ? FirstInstNum : LiveBegin[i];
- InstNumberT End = (IsGlobal && LiveOut[i]) ? LastInstNum + 1 : LiveEnd[i];
- if (Begin == Inst::NumberSentinel && End == Inst::NumberSentinel)
- continue;
- if (Begin <= FirstInstNum)
- Begin = FirstInstNum;
- if (End == Inst::NumberSentinel)
- End = LastInstNum + 1;
+ if (i == i1)
+ ++IBB;
+ if (i == i2)
+ ++IEB;
+ }
+ // Process the variables that are live across the entire block.
+ for (int i = LiveInAndOut.find_first(); i != -1;
+ i = LiveInAndOut.find_next(i)) {
Variable *Var = Liveness->getVariable(i, this);
- Liveness->addLiveRange(Var, Begin, End, 1);
+ Liveness->addLiveRange(Var, FirstInstNum, LastInstNum + 1, 1);
}
}
Str << "\n";
}
// Dump the live-in variables.
- llvm::BitVector LiveIn;
+ LivenessBV LiveIn;
if (Liveness)
LiveIn = Liveness->getLiveIn(this);
if (Func->getContext()->isVerbose(IceV_Liveness) && !LiveIn.empty()) {
I->dumpDecorated(Func);
}
// Dump the live-out variables.
- llvm::BitVector LiveOut;
+ LivenessBV LiveOut;
if (Liveness)
LiveOut = Liveness->getLiveOut(this);
if (Func->getContext()->isVerbose(IceV_Liveness) && !LiveOut.empty()) {
InstList &getInsts() { return Insts; }
void appendInst(Inst *Inst);
void renumberInstructions();
+ // Rough and generally conservative estimate of the number of
+ // instructions in the block. It is updated when an instruction is
+ // added, but not when deleted. It is recomputed during
+ // renumberInstructions().
+ InstNumberT getInstCountEstimate() const { return InstCountEstimate; }
// Add a predecessor edge to the InEdges list for each of this
// node's successors.
const SizeT Number; // label index
IceString Name; // for dumping only
bool HasReturn; // does this block need an epilog?
+ InstNumberT InstCountEstimate; // rough instruction count estimate
NodeList InEdges; // in no particular order
NodeList OutEdges; // in no particular order
PhiList Phis; // unordered set of phi instructions
// numbers are used for representing Variable live ranges.
typedef int32_t InstNumberT;
+// A LiveBeginEndMapEntry maps a Variable::Number value to an
+// Inst::Number value, giving the instruction number that begins or
+// ends a variable's live range.
+typedef std::pair<SizeT, InstNumberT> LiveBeginEndMapEntry;
+typedef std::vector<LiveBeginEndMapEntry> LiveBeginEndMap;
+typedef llvm::BitVector LivenessBV;
+
typedef uint32_t TimerStackIdT;
typedef uint32_t TimerIdT;
Inst::Inst(Cfg *Func, InstKind Kind, SizeT MaxSrcs, Variable *Dest)
: Kind(Kind), Number(Func->newInstNumber()), Deleted(false), Dead(false),
- HasSideEffects(false), Dest(Dest), MaxSrcs(MaxSrcs), NumSrcs(0),
+ HasSideEffects(false), IsDestNonKillable(false), Dest(Dest),
+ MaxSrcs(MaxSrcs), NumSrcs(0),
Srcs(Func->allocateArrayOf<Operand *>(MaxSrcs)), LiveRangesEnded(0) {}
// Assign the instruction a new number.
return false;
}
-void Inst::livenessLightweight(Cfg *Func, llvm::BitVector &Live) {
+void Inst::livenessLightweight(Cfg *Func, LivenessBV &Live) {
assert(!isDeleted());
if (llvm::isa<InstFakeKill>(this))
return;
resetLastUses();
+ VariablesMetadata *VMetadata = Func->getVMetadata();
SizeT VarIndex = 0;
for (SizeT I = 0; I < getSrcSize(); ++I) {
Operand *Src = getSrc(I);
SizeT NumVars = Src->getNumVars();
for (SizeT J = 0; J < NumVars; ++J, ++VarIndex) {
const Variable *Var = Src->getVar(J);
- if (Func->getVMetadata()->isMultiBlock(Var))
+ if (VMetadata->isMultiBlock(Var))
continue;
SizeT Index = Var->getIndex();
if (Live[Index])
}
}
-void Inst::liveness(InstNumberT InstNumber, llvm::BitVector &Live,
- Liveness *Liveness, const CfgNode *Node) {
+void Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
+ Liveness *Liveness, LiveBeginEndMap *LiveBegin,
+ LiveBeginEndMap *LiveEnd) {
assert(!isDeleted());
if (llvm::isa<InstFakeKill>(this))
return;
- std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(Node);
- std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(Node);
Dead = false;
if (Dest) {
- SizeT VarNum = Liveness->getLiveIndex(Dest);
+ SizeT VarNum = Liveness->getLiveIndex(Dest->getIndex());
if (Live[VarNum]) {
- Live[VarNum] = false;
- LiveBegin[VarNum] = InstNumber;
+ if (!isDestNonKillable()) {
+ Live[VarNum] = false;
+ if (LiveBegin) {
+ LiveBegin->push_back(std::make_pair(VarNum, InstNumber));
+ }
+ }
} else {
if (!hasSideEffects())
Dead = true;
SizeT NumVars = Src->getNumVars();
for (SizeT J = 0; J < NumVars; ++J, ++VarIndex) {
const Variable *Var = Src->getVar(J);
- SizeT VarNum = Liveness->getLiveIndex(Var);
+ SizeT VarNum = Liveness->getLiveIndex(Var->getIndex());
if (!Live[VarNum]) {
setLastUse(VarIndex);
if (!IsPhi) {
// setting it only when LiveEnd[VarNum]==0 (sentinel value).
// Note that it's OK to set LiveBegin multiple times because
// of the backwards traversal.
- if (LiveEnd[VarNum] == 0) {
- LiveEnd[VarNum] = InstNumber;
+ if (LiveEnd) {
+ // Ideally, we would verify that VarNum wasn't already
+ // added in this block, but this can't be done very
+ // efficiently with LiveEnd as a vector. Instead,
+ // livenessPostprocess() verifies this after the vector
+ // has been sorted.
+ LiveEnd->push_back(std::make_pair(VarNum, InstNumber));
}
}
}
// Updates liveness for a particular operand based on the given
// predecessor edge. Doesn't mark the operand as live if the Phi
// instruction is dead or deleted.
-void InstPhi::livenessPhiOperand(llvm::BitVector &Live, CfgNode *Target,
+void InstPhi::livenessPhiOperand(LivenessBV &Live, CfgNode *Target,
Liveness *Liveness) {
if (isDeleted() || Dead)
return;
for (SizeT I = 0; I < getSrcSize(); ++I) {
if (Labels[I] == Target) {
if (Variable *Var = llvm::dyn_cast<Variable>(getSrc(I))) {
- SizeT SrcIndex = Liveness->getLiveIndex(Var);
+ SizeT SrcIndex = Liveness->getLiveIndex(Var->getIndex());
if (!Live[SrcIndex]) {
setLastUse(I);
Live[SrcIndex] = true;
InstNumberT getNumber() const { return Number; }
void renumber(Cfg *Func);
- static const InstNumberT NumberDeleted = -1;
- static const InstNumberT NumberSentinel = 0;
+ enum { NumberDeleted = -1, NumberSentinel = 0 };
bool isDeleted() const { return Deleted; }
void setDeleted() { Deleted = true; }
bool hasSideEffects() const { return HasSideEffects; }
+ bool isDestNonKillable() const { return IsDestNonKillable; }
+ void setDestNonKillable() { IsDestNonKillable = true; }
+
Variable *getDest() const { return Dest; }
SizeT getSrcSize() const { return NumSrcs; }
virtual bool isSimpleAssign() const { return false; }
- void livenessLightweight(Cfg *Func, llvm::BitVector &Live);
- void liveness(InstNumberT InstNumber, llvm::BitVector &Live,
- Liveness *Liveness, const CfgNode *Node);
+ void livenessLightweight(Cfg *Func, LivenessBV &Live);
+ void liveness(InstNumberT InstNumber, LivenessBV &Live, Liveness *Liveness,
+ LiveBeginEndMap *LiveBegin, LiveBeginEndMap *LiveEnd);
// Get the number of native instructions that this instruction
// ultimately emits. By default, high-level instructions don't
// call or a volatile load that can't be removed even if its Dest
// variable is not live.
bool HasSideEffects;
+ // IsDestNonKillable means that liveness analysis shouldn't consider
+ // this instruction to kill the Dest variable. This is used when
+ // lowering produces two assignments to the same variable.
+ bool IsDestNonKillable;
Variable *Dest;
const SizeT MaxSrcs; // only used for assert
}
void addArgument(Operand *Source, CfgNode *Label);
Operand *getOperandForTarget(CfgNode *Target) const;
- void livenessPhiOperand(llvm::BitVector &Live, CfgNode *Target,
+ void livenessPhiOperand(LivenessBV &Live, CfgNode *Target,
Liveness *Liveness);
Inst *lower(Cfg *Func);
void dump(const Cfg *Func) const override;
// Initialize most of the container sizes.
SizeT NumVars = Func->getVariables().size();
SizeT NumNodes = Func->getNumNodes();
+ VariablesMetadata *VMetadata = Func->getVMetadata();
Nodes.resize(NumNodes);
VarToLiveMap.resize(NumVars);
if (Mode == Liveness_Intervals)
// block.
for (SizeT i = 0; i < NumVars; ++i) {
Variable *Var = Func->getVariables()[i];
- if (Func->getVMetadata()->isMultiBlock(Var)) {
+ if (VMetadata->isMultiBlock(Var)) {
++NumGlobals;
} else {
- SizeT Index = Func->getVMetadata()->getLocalUseNode(Var)->getIndex();
+ SizeT Index = VMetadata->getLocalUseNode(Var)->getIndex();
++Nodes[Index].NumLocals;
}
}
Variable *Var = Func->getVariables()[i];
SizeT VarIndex = Var->getIndex();
SizeT LiveIndex;
- if (Func->getVMetadata()->isMultiBlock(Var)) {
+ if (VMetadata->isMultiBlock(Var)) {
LiveIndex = TmpNumGlobals++;
LiveToVarMap[LiveIndex] = Var;
} else {
- SizeT NodeIndex = Func->getVMetadata()->getLocalUseNode(Var)->getIndex();
+ SizeT NodeIndex = VMetadata->getLocalUseNode(Var)->getIndex();
LiveIndex = Nodes[NodeIndex].NumLocals++;
Nodes[NodeIndex].LiveToVarMap[LiveIndex] = Var;
LiveIndex += NumGlobals;
return Nodes[NodeIndex].LiveToVarMap[LiveIndex - NumGlobals];
}
-SizeT Liveness::getLiveIndex(const Variable *Var) const {
- return VarToLiveMap[Var->getIndex()];
-}
-
void Liveness::addLiveRange(Variable *Var, InstNumberT Start, InstNumberT End,
uint32_t WeightDelta) {
LiveRange &LiveRange = LiveRanges[Var->getIndex()];
// LiveIn and LiveOut track the in- and out-liveness of the global
// variables. The size of each vector is
// LivenessNode::NumGlobals.
- llvm::BitVector LiveIn, LiveOut;
+ LivenessBV LiveIn, LiveOut;
// LiveBegin and LiveEnd track the instruction numbers of the start
// and end of each variable's live range within this block. The
- // size of each vector is NumLocals + Liveness::NumGlobals.
- std::vector<InstNumberT> LiveBegin, LiveEnd;
+ // index/key of each element is less than NumLocals +
+ // Liveness::NumGlobals.
+ LiveBeginEndMap LiveBegin, LiveEnd;
private:
// TODO: Disable these constructors when Liveness::Nodes is no
Liveness(Cfg *Func, LivenessMode Mode)
: Func(Func), Mode(Mode), NumGlobals(0) {}
void init();
+ Cfg *getFunc() const { return Func; }
+ LivenessMode getMode() const { return Mode; }
Variable *getVariable(SizeT LiveIndex, const CfgNode *Node) const;
- SizeT getLiveIndex(const Variable *Var) const;
+ SizeT getLiveIndex(SizeT VarIndex) const { return VarToLiveMap[VarIndex]; }
SizeT getNumGlobalVars() const { return NumGlobals; }
SizeT getNumVarsInNode(const CfgNode *Node) const {
return NumGlobals + Nodes[Node->getIndex()].NumLocals;
}
- llvm::BitVector &getLiveIn(const CfgNode *Node) {
+ LivenessBV &getLiveIn(const CfgNode *Node) {
return Nodes[Node->getIndex()].LiveIn;
}
- llvm::BitVector &getLiveOut(const CfgNode *Node) {
+ LivenessBV &getLiveOut(const CfgNode *Node) {
return Nodes[Node->getIndex()].LiveOut;
}
- std::vector<InstNumberT> &getLiveBegin(const CfgNode *Node) {
- return Nodes[Node->getIndex()].LiveBegin;
+ LiveBeginEndMap *getLiveBegin(const CfgNode *Node) {
+ return &Nodes[Node->getIndex()].LiveBegin;
}
- std::vector<InstNumberT> &getLiveEnd(const CfgNode *Node) {
- return Nodes[Node->getIndex()].LiveEnd;
+ LiveBeginEndMap *getLiveEnd(const CfgNode *Node) {
+ return &Nodes[Node->getIndex()].LiveEnd;
}
LiveRange &getLiveRange(Variable *Var);
void addLiveRange(Variable *Var, InstNumberT Start, InstNumberT End,
// Returns true if the live range contains the given instruction
// number. This is only used for validating the live range
-// calculation.
-bool LiveRange::containsValue(InstNumberT Value) const {
+// calculation. The IsDest argument indicates whether the Variable
+// being tested is used in the Dest position (as opposed to a Src
+// position).
+bool LiveRange::containsValue(InstNumberT Value, bool IsDest) const {
for (const RangeElementType &I : Range) {
- if (I.first <= Value && Value <= I.second)
+ if (I.first <= Value &&
+ (Value < I.second || (!IsDest && Value == I.second)))
return true;
}
return false;
void VariableTracking::markUse(const Inst *Instr, const CfgNode *Node,
bool IsFromDef, bool IsImplicit) {
+ if (MultiBlock == MBS_MultiBlock)
+ return;
// TODO(stichnot): If the use occurs as a source operand in the
// first instruction of the block, and its definition is in this
// block's only predecessor, we might consider not marking this as a
Metadata[Var->getIndex()].markUse(NoInst, EntryNode, IsFromDef, IsImplicit);
}
- SizeT NumNodes = Func->getNumNodes();
- for (SizeT N = 0; N < NumNodes; ++N) {
- CfgNode *Node = Func->getNodes()[N];
+ for (CfgNode *Node : Func->getNodes()) {
for (Inst *I : Node->getInsts()) {
if (I->isDeleted())
continue;
bool endsBefore(const LiveRange &Other) const;
bool overlaps(const LiveRange &Other, bool UseTrimmed = false) const;
bool overlapsInst(InstNumberT OtherBegin, bool UseTrimmed = false) const;
- bool containsValue(InstNumberT Value) const;
+ bool containsValue(InstNumberT Value, bool IsDest) const;
bool isEmpty() const { return Range.empty(); }
bool isNonpoints() const { return IsNonpoints; }
InstNumberT getStart() const {
bool getIsImplicitArg() const { return IsImplicitArgument; }
void setIsImplicitArg(bool Val = true) { IsImplicitArgument = Val; }
+ void setIgnoreLiveness() { IgnoreLiveness = true; }
+ bool getIgnoreLiveness() const { return IgnoreLiveness; }
+
int32_t getStackOffset() const { return StackOffset; }
void setStackOffset(int32_t Offset) { StackOffset = Offset; }
protected:
Variable(OperandKind K, Type Ty, SizeT Index, const IceString &Name)
: Operand(K, Ty), Number(Index), Name(Name), IsArgument(false),
- IsImplicitArgument(false), StackOffset(0), RegNum(NoRegister),
- RegNumTmp(NoRegister), Weight(1), LoVar(NULL), HiVar(NULL) {
+ IsImplicitArgument(false), IgnoreLiveness(false), StackOffset(0),
+ RegNum(NoRegister), RegNumTmp(NoRegister), Weight(1), LoVar(NULL),
+ HiVar(NULL) {
Vars = VarsReal;
Vars[0] = this;
NumVars = 1;
IceString Name;
bool IsArgument;
bool IsImplicitArgument;
+ // IgnoreLiveness means that the variable should be ignored when
+ // constructing and validating live ranges. This is usually
+ // reserved for the stack pointer.
+ bool IgnoreLiveness;
// StackOffset is the canonical location on stack (only if
// RegNum==NoRegister || IsArgument).
int32_t StackOffset;
PhysicalRegisters[RegNum] = Reg;
// Specially mark esp as an "argument" so that it is considered
// live upon function entry.
- if (RegNum == RegX8632::Reg_esp)
+ if (RegNum == RegX8632::Reg_esp) {
Func->addImplicitArg(Reg);
+ Reg->setIgnoreLiveness();
+ }
}
return Reg;
}
_shl(T_2, T_1);
_test(T_1, BitTest);
_br(CondX86::Br_e, Label);
- // Because of the intra-block control flow, we need to fake a use
- // of T_3 to prevent its earlier definition from being dead-code
- // eliminated in the presence of its later definition.
- Context.insert(InstFakeUse::create(Func, T_3));
- _mov(T_3, T_2);
- _mov(T_2, Zero);
+ // T_2 and T_3 are being assigned again because of the
+ // intra-block control flow, so we need the _mov_nonkillable
+ // variant to avoid liveness problems.
+ _mov_nonkillable(T_3, T_2);
+ _mov_nonkillable(T_2, Zero);
Context.insert(Label);
_mov(DestLo, T_2);
_mov(DestHi, T_3);
_shr(T_3, T_1);
_test(T_1, BitTest);
_br(CondX86::Br_e, Label);
- // Because of the intra-block control flow, we need to fake a use
- // of T_3 to prevent its earlier definition from being dead-code
- // eliminated in the presence of its later definition.
- Context.insert(InstFakeUse::create(Func, T_2));
- _mov(T_2, T_3);
- _mov(T_3, Zero);
+ // T_2 and T_3 are being assigned again because of the
+ // intra-block control flow, so we need the _mov_nonkillable
+ // variant to avoid liveness problems.
+ _mov_nonkillable(T_2, T_3);
+ _mov_nonkillable(T_3, Zero);
Context.insert(Label);
_mov(DestLo, T_2);
_mov(DestHi, T_3);
_sar(T_3, T_1);
_test(T_1, BitTest);
_br(CondX86::Br_e, Label);
- // Because of the intra-block control flow, we need to fake a use
- // of T_3 to prevent its earlier definition from being dead-code
- // eliminated in the presence of its later definition.
- Context.insert(InstFakeUse::create(Func, T_2));
- _mov(T_2, T_3);
+ // T_2 and T_3 are being assigned again because of the
+ // intra-block control flow, so T_2 needs the _mov_nonkillable
+ // variant to avoid liveness problems. T_3 doesn't need special
+ // treatment because it is reassigned via _sar instead of _mov.
+ _mov_nonkillable(T_2, T_3);
_sar(T_3, SignExtend);
Context.insert(Label);
_mov(DestLo, T_2);
if (HasC2) {
_br(TableFcmp[Index].C2, Label);
}
- Context.insert(InstFakeUse::create(Func, Dest));
Constant *NonDefault =
Ctx->getConstantInt32(IceType_i32, !TableFcmp[Index].Default);
- _mov(Dest, NonDefault);
+ _mov_nonkillable(Dest, NonDefault);
Context.insert(Label);
}
}
_br(CondX86::Br_ne, Label);
_cmp(Src0HiRM, Src1HiRI);
_br(CondX86::Br_ne, Label);
- Context.insert(InstFakeUse::create(Func, Dest));
- _mov(Dest, (Condition == InstIcmp::Eq ? One : Zero));
+ _mov_nonkillable(Dest, (Condition == InstIcmp::Eq ? One : Zero));
Context.insert(Label);
} else {
InstX8632Label *LabelFalse = InstX8632Label::create(Func, this);
_cmp(Src0LoRM, Src1LoRI);
_br(TableIcmp64[Index].C3, LabelTrue);
Context.insert(LabelFalse);
- Context.insert(InstFakeUse::create(Func, Dest));
- _mov(Dest, Zero);
+ _mov_nonkillable(Dest, Zero);
Context.insert(LabelTrue);
}
return;
_cmp(Src0RM, Src1);
_mov(Dest, One);
_br(getIcmp32Mapping(Inst->getCondition()), Label);
- Context.insert(InstFakeUse::create(Func, Dest));
- _mov(Dest, Zero);
+ _mov_nonkillable(Dest, Zero);
Context.insert(Label);
}
}
case Intrinsics::Stackrestore: {
Variable *esp = Func->getTarget()->getPhysicalRegister(RegX8632::Reg_esp);
- _mov(esp, Instr->getArg(0));
+ _mov_nonkillable(esp, Instr->getArg(0));
return;
}
case Intrinsics::Trap:
_mov(DestLo, SrcLoRI);
_mov(DestHi, SrcHiRI);
_br(CondX86::Br_ne, Label);
- Context.insert(InstFakeUse::create(Func, DestLo));
- Context.insert(InstFakeUse::create(Func, DestHi));
Operand *SrcFLo = loOperand(SrcF);
Operand *SrcFHi = hiOperand(SrcF);
SrcLoRI = legalize(SrcFLo, Legal_Reg | Legal_Imm);
SrcHiRI = legalize(SrcFHi, Legal_Reg | Legal_Imm);
- _mov(DestLo, SrcLoRI);
- _mov(DestHi, SrcHiRI);
+ _mov_nonkillable(DestLo, SrcLoRI);
+ _mov_nonkillable(DestHi, SrcHiRI);
} else {
_cmp(ConditionRM, Zero);
SrcT = legalize(SrcT, Legal_Reg | Legal_Imm);
_mov(Dest, SrcT);
_br(CondX86::Br_ne, Label);
- Context.insert(InstFakeUse::create(Func, Dest));
SrcF = legalize(SrcF, Legal_Reg | Legal_Imm);
- _mov(Dest, SrcF);
+ _mov_nonkillable(Dest, SrcF);
}
Context.insert(Label);
}
void TargetX8632::postLower() {
- if (Ctx->getOptLevel() != Opt_m1)
+ if (Ctx->getOptLevel() != Opt_m1) {
+ // Find two-address non-SSA instructions where Dest==Src0, and set
+ // the DestNonKillable flag to keep liveness analysis consistent.
+ for (Inst *Inst : Context) {
+ if (Inst->isDeleted())
+ continue;
+ if (Variable *Dest = Inst->getDest()) {
+ // TODO(stichnot): We may need to consider all source
+ // operands, not just the first one, if using 3-address
+ // instructions.
+ if (Inst->getSrcSize() > 0 && Inst->getSrc(0) == Dest)
+ Inst->setDestNonKillable();
+ }
+ }
return;
- TimerMarker T(TimerStack::TT_postLower, Func);
+ }
// TODO: Avoid recomputing WhiteList every instruction.
RegSetMask RegInclude = RegSet_All;
RegSetMask RegExclude = RegSet_StackPointer;
// Mark eax as possibly modified by cmpxchg.
Context.insert(
InstFakeDef::create(Func, Eax, llvm::dyn_cast<Variable>(DestOrAddr)));
+ _set_dest_nonkillable();
}
void _cmpxchg8b(OperandX8632Mem *Addr, Variable *Edx, Variable *Eax,
Variable *Ecx, Variable *Ebx, bool Locked) {
InstX8632Cmpxchg8b::create(Func, Addr, Edx, Eax, Ecx, Ebx, Locked));
// Mark edx, and eax as possibly modified by cmpxchg8b.
Context.insert(InstFakeDef::create(Func, Edx));
+ _set_dest_nonkillable();
Context.insert(InstFakeDef::create(Func, Eax));
+ _set_dest_nonkillable();
}
void _cvt(Variable *Dest, Operand *Src0, InstX8632Cvt::CvtVariant Variant) {
Context.insert(InstX8632Cvt::create(Func, Dest, Src0, Variant));
Dest = makeReg(Src0->getType(), RegNum);
Context.insert(InstX8632Mov::create(Func, Dest, Src0));
}
+ void _mov_nonkillable(Variable *Dest, Operand *Src0) {
+ Inst *NewInst = InstX8632Mov::create(Func, Dest, Src0);
+ NewInst->setDestNonKillable();
+ Context.insert(NewInst);
+ }
void _movd(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Movd::create(Func, Dest, Src0));
}
// Model that update with a FakeDef.
Context.insert(
InstFakeDef::create(Func, Src, llvm::dyn_cast<Variable>(Dest)));
+ _set_dest_nonkillable();
}
void _xchg(Operand *Dest, Variable *Src) {
Context.insert(InstX8632Xchg::create(Func, Dest, Src));
// The xchg modifies Dest and Src -- model that update with a FakeDef.
Context.insert(
InstFakeDef::create(Func, Src, llvm::dyn_cast<Variable>(Dest)));
+ _set_dest_nonkillable();
}
void _xor(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Xor::create(Func, Dest, Src0));
}
+ void _set_dest_nonkillable() {
+ Context.getLastInserted()->setDestNonKillable();
+ }
const X86InstructionSet InstructionSet;
bool IsEbpBasedFrame;
// elements and to the flat element for the top of the stack.
double Current = timestamp();
double Delta = Current - LastTimestamp;
- LastTimestamp = Current;
if (StackTop) {
TimerIdT Leaf = Nodes[StackTop].Interior;
if (Leaf >= LeafTimes.size())
assert(Next < Prefix);
Prefix = Next;
}
+ // Capture the next timestamp *after* the updates are finished.
+ // This minimizes how much the timer can perturb the reported
+ // timing. The numbers may not sum to 100%, and the missing amount
+ // is indicative of the overhead of timing.
+ LastTimestamp = timestamp();
}
void TimerStack::reset() {
X(parse) \
X(placePhiLoads) \
X(placePhiStores) \
- X(postLower) \
X(regAlloc) \
X(renumberInstructions) \
X(szmain) \
; OPTM1: test {{.*}}, 32
; OPTM1: je
+define internal i32 @shl64BitSignedTrunc(i64 %a, i64 %b) {
+entry:
+ %shl = shl i64 %a, %b
+ %result = trunc i64 %shl to i32
+ ret i32 %result
+}
+; CHECK-LABEL: shl64BitSignedTrunc
+; CHECK: mov
+; CHECK: shl e
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shl64BitSignedTrunc
+; OPTM1: shld
+; OPTM1: shl e
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+
define internal i64 @shl64BitUnsigned(i64 %a, i64 %b) {
entry:
%shl = shl i64 %a, %b
; OPTM1: je
; OPTM1: sar {{.*}}, 31
+define internal i32 @shr64BitSignedTrunc(i64 %a, i64 %b) {
+entry:
+ %shr = ashr i64 %a, %b
+ %result = trunc i64 %shr to i32
+ ret i32 %result
+}
+; CHECK-LABEL: shr64BitSignedTrunc
+; CHECK: shrd
+; CHECK: sar
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shr64BitSignedTrunc
+; OPTM1: shrd
+; OPTM1: sar
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+; OPTM1: sar {{.*}}, 31
+
define internal i64 @shr64BitUnsigned(i64 %a, i64 %b) {
entry:
%shr = lshr i64 %a, %b
; OPTM1: test {{.*}}, 32
; OPTM1: je
+define internal i32 @shr64BitUnsignedTrunc(i64 %a, i64 %b) {
+entry:
+ %shr = lshr i64 %a, %b
+ %result = trunc i64 %shr to i32
+ ret i32 %result
+}
+; CHECK-LABEL: shr64BitUnsignedTrunc
+; CHECK: shrd
+; CHECK: shr
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shr64BitUnsignedTrunc
+; OPTM1: shrd
+; OPTM1: shr
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+
define internal i64 @and64BitSigned(i64 %a, i64 %b) {
entry:
%and = and i64 %b, %a
OSDN Git Service
