#include "llvm/Support/raw_ostream.h"
using namespace llvm;
+#define DEBUG_TYPE "legalize-types"
+
static cl::opt<bool>
EnableExpensiveChecks("enable-legalize-types-checking", cl::Hidden);
// (for example because it was created but not used). In general, we cannot
// distinguish between new nodes and deleted nodes.
SmallVector<SDNode*, 16> NewNodes;
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
+ for (SDNode &Node : DAG.allnodes()) {
// Remember nodes marked NewNode - they are subject to extra checking below.
- if (I->getNodeId() == NewNode)
- NewNodes.push_back(I);
+ if (Node.getNodeId() == NewNode)
+ NewNodes.push_back(&Node);
- for (unsigned i = 0, e = I->getNumValues(); i != e; ++i) {
- SDValue Res(I, i);
+ for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) {
+ SDValue Res(&Node, i);
bool Failed = false;
unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) {
Mapped |= 1;
// Check that remapped values are only used by nodes marked NewNode.
- for (SDNode::use_iterator UI = I->use_begin(), UE = I->use_end();
+ for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
UI != UE; ++UI)
if (UI.getUse().getResNo() == i)
assert(UI->getNodeId() == NewNode &&
if (WidenedVectors.find(Res) != WidenedVectors.end())
Mapped |= 128;
- if (I->getNodeId() != Processed) {
+ if (Node.getNodeId() != Processed) {
// Since we allow ReplacedValues to map deleted nodes, it may map nodes
// marked NewNode too, since a deleted node may have been reallocated as
// another node that has not been seen by the LegalizeTypes machinery.
- if ((I->getNodeId() == NewNode && Mapped > 1) ||
- (I->getNodeId() != NewNode && Mapped != 0)) {
+ if ((Node.getNodeId() == NewNode && Mapped > 1) ||
+ (Node.getNodeId() != NewNode && Mapped != 0)) {
dbgs() << "Unprocessed value in a map!";
Failed = true;
}
- } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(I)) {
+ } else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(&Node)) {
if (Mapped > 1) {
dbgs() << "Value with legal type was transformed!";
Failed = true;
if (Mapped & 128)
dbgs() << " WidenedVectors";
dbgs() << "\n";
- llvm_unreachable(0);
+ llvm_unreachable(nullptr);
}
}
}
// Walk all nodes in the graph, assigning them a NodeId of 'ReadyToProcess'
// (and remembering them) if they are leaves and assigning 'Unanalyzed' if
// non-leaves.
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
- if (I->getNumOperands() == 0) {
- I->setNodeId(ReadyToProcess);
- Worklist.push_back(I);
+ for (SDNode &Node : DAG.allnodes()) {
+ if (Node.getNumOperands() == 0) {
+ Node.setNodeId(ReadyToProcess);
+ Worklist.push_back(&Node);
} else {
- I->setNodeId(Unanalyzed);
+ Node.setNodeId(Unanalyzed);
}
}
Changed = true;
goto NodeDone;
case TargetLowering::TypeSoftenFloat:
- SoftenFloatResult(N, i);
- Changed = true;
- goto NodeDone;
+ Changed = SoftenFloatResult(N, i);
+ if (Changed)
+ goto NodeDone;
+ // If not changed, the result type should be legally in register.
+ assert(isLegalInHWReg(ResultVT) &&
+ "Unchanged SoftenFloatResult should be legal in register!");
+ goto ScanOperands;
case TargetLowering::TypeExpandFloat:
ExpandFloatResult(N, i);
Changed = true;
WidenVectorResult(N, i);
Changed = true;
goto NodeDone;
+ case TargetLowering::TypePromoteFloat:
+ PromoteFloatResult(N, i);
+ Changed = true;
+ goto NodeDone;
}
}
NeedsReanalyzing = WidenVectorOperand(N, i);
Changed = true;
break;
+ case TargetLowering::TypePromoteFloat:
+ NeedsReanalyzing = PromoteFloatOperand(N, i);
+ Changed = true;
+ break;
}
break;
}
// In a debug build, scan all the nodes to make sure we found them all. This
// ensures that there are no cycles and that everything got processed.
#ifndef NDEBUG
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
+ for (SDNode &Node : DAG.allnodes()) {
bool Failed = false;
// Check that all result types are legal.
- if (!IgnoreNodeResults(I))
- for (unsigned i = 0, NumVals = I->getNumValues(); i < NumVals; ++i)
- if (!isTypeLegal(I->getValueType(i))) {
- dbgs() << "Result type " << i << " illegal!\n";
+ // A value type is illegal if its TypeAction is not TypeLegal,
+ // and TLI.RegClassForVT does not have a register class for this type.
+ // For example, the x86_64 target has f128 that is not TypeLegal,
+ // to have softened operators, but it also has FR128 register class to
+ // pass and return f128 values. Hence a legalized node can have f128 type.
+ if (!IgnoreNodeResults(&Node))
+ for (unsigned i = 0, NumVals = Node.getNumValues(); i < NumVals; ++i)
+ if (!isTypeLegal(Node.getValueType(i)) &&
+ !TLI.isTypeLegal(Node.getValueType(i))) {
+ dbgs() << "Result type " << i << " illegal: ";
+ Node.dump();
Failed = true;
}
// Check that all operand types are legal.
- for (unsigned i = 0, NumOps = I->getNumOperands(); i < NumOps; ++i)
- if (!IgnoreNodeResults(I->getOperand(i).getNode()) &&
- !isTypeLegal(I->getOperand(i).getValueType())) {
- dbgs() << "Operand type " << i << " illegal!\n";
+ for (unsigned i = 0, NumOps = Node.getNumOperands(); i < NumOps; ++i)
+ if (!IgnoreNodeResults(Node.getOperand(i).getNode()) &&
+ !isTypeLegal(Node.getOperand(i).getValueType()) &&
+ !TLI.isTypeLegal(Node.getOperand(i).getValueType())) {
+ dbgs() << "Operand type " << i << " illegal: ";
+ Node.getOperand(i).dump();
Failed = true;
}
- if (I->getNodeId() != Processed) {
- if (I->getNodeId() == NewNode)
+ if (Node.getNodeId() != Processed) {
+ if (Node.getNodeId() == NewNode)
dbgs() << "New node not analyzed?\n";
- else if (I->getNodeId() == Unanalyzed)
+ else if (Node.getNodeId() == Unanalyzed)
dbgs() << "Unanalyzed node not noticed?\n";
- else if (I->getNodeId() > 0)
+ else if (Node.getNodeId() > 0)
dbgs() << "Operand not processed?\n";
- else if (I->getNodeId() == ReadyToProcess)
+ else if (Node.getNodeId() == ReadyToProcess)
dbgs() << "Not added to worklist?\n";
Failed = true;
}
if (Failed) {
- I->dump(&DAG); dbgs() << "\n";
- llvm_unreachable(0);
+ Node.dump(&DAG); dbgs() << "\n";
+ llvm_unreachable(nullptr);
}
}
#endif
// Some operands changed - update the node.
if (!NewOps.empty()) {
- SDNode *M = DAG.UpdateNodeOperands(N, &NewOps[0], NewOps.size());
+ SDNode *M = DAG.UpdateNodeOperands(N, NewOps);
if (M != N) {
// The node morphed into a different node. Normally for this to happen
// the original node would have to be marked NewNode. However this can
// replaced with other values.
RemapValue(I->second);
N = I->second;
- assert(N.getNode()->getNodeId() != NewNode && "Mapped to new node!");
+
+ // Note that it is possible to have N.getNode()->getNodeId() == NewNode at
+ // this point because it is possible for a node to be put in the map before
+ // being processed.
}
}
: SelectionDAG::DAGUpdateListener(dtl.getDAG()),
DTL(dtl), NodesToAnalyze(nta) {}
- virtual void NodeDeleted(SDNode *N, SDNode *E) {
+ void NodeDeleted(SDNode *N, SDNode *E) override {
assert(N->getNodeId() != DAGTypeLegalizer::ReadyToProcess &&
N->getNodeId() != DAGTypeLegalizer::Processed &&
"Invalid node ID for RAUW deletion!");
NodesToAnalyze.insert(E);
}
- virtual void NodeUpdated(SDNode *N) {
+ void NodeUpdated(SDNode *N) override {
// Node updates can mean pretty much anything. It is possible that an
// operand was set to something already processed (f.e.) in which case
// this node could become ready. Recompute its flags.
AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedIntegers[Op];
- assert(OpEntry.getNode() == 0 && "Node is already promoted!");
+ assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
- assert(Result.getValueType() ==
- TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ // f128 of x86_64 could be kept in SSE registers,
+ // but sometimes softened to i128.
+ assert((Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) ||
+ Op.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType())) &&
"Invalid type for softened float");
AnalyzeNewValue(Result);
SDValue &OpEntry = SoftenedFloats[Op];
- assert(OpEntry.getNode() == 0 && "Node is already converted to integer!");
+ // Allow repeated calls to save f128 type nodes
+ // or any node with type that transforms to itself.
+ // Many operations on these types are not softened.
+ assert((!OpEntry.getNode()||
+ Op.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType())) &&
+ "Node is already converted to integer!");
+ OpEntry = Result;
+}
+
+void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
+ assert(Result.getValueType() ==
+ TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
+ "Invalid type for promoted float");
+ AnalyzeNewValue(Result);
+
+ SDValue &OpEntry = PromotedFloats[Op];
+ assert(!OpEntry.getNode() && "Node is already promoted!");
OpEntry = Result;
}
AnalyzeNewValue(Result);
SDValue &OpEntry = ScalarizedVectors[Op];
- assert(OpEntry.getNode() == 0 && "Node is already scalarized!");
+ assert(!OpEntry.getNode() && "Node is already scalarized!");
OpEntry = Result;
}
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op];
- assert(Entry.first.getNode() == 0 && "Node already expanded");
+ assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op];
- assert(Entry.first.getNode() == 0 && "Node already expanded");
+ assert(!Entry.first.getNode() && "Node already expanded");
Entry.first = Lo;
Entry.second = Hi;
}
// Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op];
- assert(Entry.first.getNode() == 0 && "Node already split");
+ assert(!Entry.first.getNode() && "Node already split");
Entry.first = Lo;
Entry.second = Hi;
}
AnalyzeNewValue(Result);
SDValue &OpEntry = WidenedVectors[Op];
- assert(OpEntry.getNode() == 0 && "Node already widened!");
+ assert(!OpEntry.getNode() && "Node already widened!");
OpEntry = Result;
}
/// BitConvertToInteger - Convert to an integer of the same size.
SDValue DAGTypeLegalizer::BitConvertToInteger(SDValue Op) {
unsigned BitWidth = Op.getValueType().getSizeInBits();
- return DAG.getNode(ISD::BITCAST, Op.getDebugLoc(),
+ return DAG.getNode(ISD::BITCAST, SDLoc(Op),
EVT::getIntegerVT(*DAG.getContext(), BitWidth), Op);
}
unsigned EltWidth = Op.getValueType().getVectorElementType().getSizeInBits();
EVT EltNVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
unsigned NumElts = Op.getValueType().getVectorNumElements();
- return DAG.getNode(ISD::BITCAST, Op.getDebugLoc(),
+ return DAG.getNode(ISD::BITCAST, SDLoc(Op),
EVT::getVectorVT(*DAG.getContext(), EltNVT, NumElts), Op);
}
SDValue DAGTypeLegalizer::CreateStackStoreLoad(SDValue Op,
EVT DestVT) {
- DebugLoc dl = Op.getDebugLoc();
+ SDLoc dl(Op);
// Create the stack frame object. Make sure it is aligned for both
// the source and destination types.
SDValue StackPtr = DAG.CreateStackTemporary(Op.getValueType(), DestVT);
// The target didn't want to custom lower it after all.
return false;
+ // When called from DAGTypeLegalizer::ExpandIntegerResult, we might need to
+ // provide the same kind of custom splitting behavior.
+ if (Results.size() == N->getNumValues() + 1 && LegalizeResult) {
+ // We've legalized a return type by splitting it. If there is a chain,
+ // replace that too.
+ SetExpandedInteger(SDValue(N, 0), Results[0], Results[1]);
+ if (N->getNumValues() > 1)
+ ReplaceValueWith(SDValue(N, 1), Results[2]);
+ return true;
+ }
+
// Make everything that once used N's values now use those in Results instead.
assert(Results.size() == N->getNumValues() &&
"Custom lowering returned the wrong number of results!");
- for (unsigned i = 0, e = Results.size(); i != e; ++i)
+ for (unsigned i = 0, e = Results.size(); i != e; ++i) {
ReplaceValueWith(SDValue(N, i), Results[i]);
+ }
return true;
}
return SDValue(N->getOperand(ResNo));
}
-/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
-/// which is split into two not necessarily identical pieces.
-void DAGTypeLegalizer::GetSplitDestVTs(EVT InVT, EVT &LoVT, EVT &HiVT) {
- // Currently all types are split in half.
- if (!InVT.isVector()) {
- LoVT = HiVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
- } else {
- unsigned NumElements = InVT.getVectorNumElements();
- assert(!(NumElements & 1) && "Splitting vector, but not in half!");
- LoVT = HiVT = EVT::getVectorVT(*DAG.getContext(),
- InVT.getVectorElementType(), NumElements/2);
- }
-}
-
/// GetPairElements - Use ISD::EXTRACT_ELEMENT nodes to extract the low and
/// high parts of the given value.
void DAGTypeLegalizer::GetPairElements(SDValue Pair,
SDValue &Lo, SDValue &Hi) {
- DebugLoc dl = Pair.getDebugLoc();
+ SDLoc dl(Pair);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
- DAG.getIntPtrConstant(0));
+ DAG.getIntPtrConstant(0, dl));
Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
- DAG.getIntPtrConstant(1));
+ DAG.getIntPtrConstant(1, dl));
}
SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, EVT EltVT,
SDValue Index) {
- DebugLoc dl = Index.getDebugLoc();
+ SDLoc dl(Index);
// Make sure the index type is big enough to compute in.
- if (Index.getValueType().bitsGT(TLI.getPointerTy()))
- Index = DAG.getNode(ISD::TRUNCATE, dl, TLI.getPointerTy(), Index);
- else
- Index = DAG.getNode(ISD::ZERO_EXTEND, dl, TLI.getPointerTy(), Index);
+ Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy(DAG.getDataLayout()));
// Calculate the element offset and add it to the pointer.
unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
+ assert(EltSize * 8 == EltVT.getSizeInBits() &&
+ "Converting bits to bytes lost precision");
Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
- DAG.getConstant(EltSize, Index.getValueType()));
+ DAG.getConstant(EltSize, dl, Index.getValueType()));
return DAG.getNode(ISD::ADD, dl, Index.getValueType(), Index, VecPtr);
}
/// JoinIntegers - Build an integer with low bits Lo and high bits Hi.
SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
- // Arbitrarily use dlHi for result DebugLoc
- DebugLoc dlHi = Hi.getDebugLoc();
- DebugLoc dlLo = Lo.getDebugLoc();
+ // Arbitrarily use dlHi for result SDLoc
+ SDLoc dlHi(Hi);
+ SDLoc dlLo(Lo);
EVT LVT = Lo.getValueType();
EVT HVT = Hi.getValueType();
EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo);
Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi);
Hi = DAG.getNode(ISD::SHL, dlHi, NVT, Hi,
- DAG.getConstant(LVT.getSizeInBits(), TLI.getPointerTy()));
+ DAG.getConstant(LVT.getSizeInBits(), dlHi,
+ TLI.getPointerTy(DAG.getDataLayout())));
return DAG.getNode(ISD::OR, dlHi, NVT, Lo, Hi);
}
SDValue DAGTypeLegalizer::LibCallify(RTLIB::Libcall LC, SDNode *N,
bool isSigned) {
unsigned NumOps = N->getNumOperands();
- DebugLoc dl = N->getDebugLoc();
+ SDLoc dl(N);
if (NumOps == 0) {
- return TLI.makeLibCall(DAG, LC, N->getValueType(0), 0, 0, isSigned, dl);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), None, isSigned,
+ dl).first;
} else if (NumOps == 1) {
SDValue Op = N->getOperand(0);
- return TLI.makeLibCall(DAG, LC, N->getValueType(0), &Op, 1, isSigned, dl);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), Op, isSigned,
+ dl).first;
} else if (NumOps == 2) {
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
- return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, 2, isSigned, dl);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, isSigned,
+ dl).first;
}
SmallVector<SDValue, 8> Ops(NumOps);
for (unsigned i = 0; i < NumOps; ++i)
Ops[i] = N->getOperand(i);
- return TLI.makeLibCall(DAG, LC, N->getValueType(0),
- &Ops[0], NumOps, isSigned, dl);
+ return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, isSigned, dl).first;
}
// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to
Args.push_back(Entry);
}
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
- TLI.getPointerTy());
+ TLI.getPointerTy(DAG.getDataLayout()));
Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
- TargetLowering::
- CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false,
- 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false,
- /*doesNotReturn=*/false, /*isReturnValueUsed=*/true,
- Callee, Args, DAG, Node->getDebugLoc());
+
+ TargetLowering::CallLoweringInfo CLI(DAG);
+ CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
+ .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
+ .setSExtResult(isSigned).setZExtResult(!isSigned);
+
std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
return CallInfo;
/// PromoteTargetBoolean - Promote the given target boolean to a target boolean
/// of the given type. A target boolean is an integer value, not necessarily of
/// type i1, the bits of which conform to getBooleanContents.
-SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT VT) {
- DebugLoc dl = Bool.getDebugLoc();
+///
+/// ValVT is the type of values that produced the boolean.
+SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
+ SDLoc dl(Bool);
+ EVT BoolVT = getSetCCResultType(ValVT);
ISD::NodeType ExtendCode =
- TargetLowering::getExtendForContent(TLI.getBooleanContents(VT.isVector()));
- return DAG.getNode(ExtendCode, dl, VT, Bool);
+ TargetLowering::getExtendForContent(TLI.getBooleanContents(ValVT));
+ return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
+}
+
+/// WidenTargetBoolean - Widen the given target boolean to a target boolean
+/// of the given type. The boolean vector is widened and then promoted to match
+/// the target boolean type of the given ValVT.
+SDValue DAGTypeLegalizer::WidenTargetBoolean(SDValue Bool, EVT ValVT,
+ bool WithZeroes) {
+ SDLoc dl(Bool);
+ EVT BoolVT = Bool.getValueType();
+
+ assert(ValVT.getVectorNumElements() > BoolVT.getVectorNumElements() &&
+ TLI.isTypeLegal(ValVT) &&
+ "Unexpected types in WidenTargetBoolean");
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), BoolVT.getScalarType(),
+ ValVT.getVectorNumElements());
+ Bool = ModifyToType(Bool, WideVT, WithZeroes);
+ return PromoteTargetBoolean(Bool, ValVT);
}
/// SplitInteger - Return the lower LoVT bits of Op in Lo and the upper HiVT
void DAGTypeLegalizer::SplitInteger(SDValue Op,
EVT LoVT, EVT HiVT,
SDValue &Lo, SDValue &Hi) {
- DebugLoc dl = Op.getDebugLoc();
+ SDLoc dl(Op);
assert(LoVT.getSizeInBits() + HiVT.getSizeInBits() ==
Op.getValueType().getSizeInBits() && "Invalid integer splitting!");
Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Op);
Hi = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
- DAG.getConstant(LoVT.getSizeInBits(), TLI.getPointerTy()));
+ DAG.getConstant(LoVT.getSizeInBits(), dl,
+ TLI.getPointerTy(DAG.getDataLayout())));
Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
}
OSDN Git Service
