//===----------------------------------------------------------------------===//
void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
- DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
+ dbgs() << "\n");
SDValue R = SDValue();
switch (N->getOpcode()) {
ScalarBool = TargetLowering::UndefinedBooleanContent;
}
+ EVT CondVT = Cond.getValueType();
if (ScalarBool != VecBool) {
- EVT CondVT = Cond.getValueType();
switch (ScalarBool) {
case TargetLowering::UndefinedBooleanContent:
break;
}
}
+ // Truncate the condition if needed
+ auto BoolVT = getSetCCResultType(CondVT);
+ if (BoolVT.bitsLT(CondVT))
+ Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
+
return DAG.getSelect(SDLoc(N),
LHS.getValueType(), Cond, LHS,
GetScalarizedVector(N->getOperand(2)));
//===----------------------------------------------------------------------===//
bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
- DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
+ dbgs() << "\n");
SDValue Res = SDValue();
if (!Res.getNode()) {
N->getValueType(0).getScalarType(), Elt);
// Revectorize the result so the types line up with what the uses of this
// expression expect.
- return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Op);
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
}
/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
/// invalid operands or may have other results that need legalization, we just
/// know that (at least) one result needs vector splitting.
void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
- DEBUG(dbgs() << "Split node result: ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
SDValue Lo, Hi;
// See if the target wants to custom expand this node.
case ISD::FMA:
SplitVecRes_TernaryOp(N, Lo, Hi);
break;
+ case ISD::STRICT_FADD:
+ case ISD::STRICT_FSUB:
+ case ISD::STRICT_FMUL:
+ case ISD::STRICT_FDIV:
+ case ISD::STRICT_FSQRT:
+ case ISD::STRICT_FMA:
+ case ISD::STRICT_FPOW:
+ case ISD::STRICT_FPOWI:
+ case ISD::STRICT_FSIN:
+ case ISD::STRICT_FCOS:
+ case ISD::STRICT_FEXP:
+ case ISD::STRICT_FEXP2:
+ case ISD::STRICT_FLOG:
+ case ISD::STRICT_FLOG10:
+ case ISD::STRICT_FLOG2:
+ case ISD::STRICT_FRINT:
+ case ISD::STRICT_FNEARBYINT:
+ SplitVecRes_StrictFPOp(N, Lo, Hi);
+ break;
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
}
+void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ unsigned NumOps = N->getNumOperands();
+ SDValue Chain = N->getOperand(0);
+ EVT LoVT, HiVT;
+ SDLoc dl(N);
+ std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
+
+ SmallVector<SDValue, 4> OpsLo;
+ SmallVector<SDValue, 4> OpsHi;
+
+ // The Chain is the first operand.
+ OpsLo.push_back(Chain);
+ OpsHi.push_back(Chain);
+
+ // Now process the remaining operands.
+ for (unsigned i = 1; i < NumOps; ++i) {
+ SDValue Op = N->getOperand(i);
+ SDValue OpLo = Op;
+ SDValue OpHi = Op;
+
+ EVT InVT = Op.getValueType();
+ if (InVT.isVector()) {
+ // If the input also splits, handle it directly for a
+ // compile time speedup. Otherwise split it by hand.
+ if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
+ GetSplitVector(Op, OpLo, OpHi);
+ else
+ std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
+ }
+
+ OpsLo.push_back(OpLo);
+ OpsHi.push_back(OpHi);
+ }
+
+ EVT LoValueVTs[] = {LoVT, MVT::Other};
+ EVT HiValueVTs[] = {HiVT, MVT::Other};
+ Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
+ Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
+
+ // Build a factor node to remember that this Op is independent of the
+ // other one.
+ Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+ Lo.getValue(1), Hi.getValue(1));
+
+ // Legalize the chain result - switch anything that used the old chain to
+ // use the new one.
+ ReplaceValueWith(SDValue(N, 1), Chain);
+}
+
void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
SDValue &Hi) {
SDValue Vec = N->getOperand(0);
Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
MLD->isExpandingLoad());
+ unsigned HiOffset = LoMemVT.getStoreSize();
- MMO = DAG.getMachineFunction().
- getMachineMemOperand(MLD->getPointerInfo(),
- MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
- SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
+ MMO = DAG.getMachineFunction().getMachineMemOperand(
+ MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
+ HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(),
+ MLD->getRanges());
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
ExtType, MLD->isExpandingLoad());
-
// Build a factor node to remember that this load is independent of the
// other one.
Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
- DEBUG(dbgs() << "Split vector extend via incremental extend:";
- N->dump(&DAG); dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
+ N->dump(&DAG); dbgs() << "\n");
// Extend the source vector by one step.
SDValue NewSrc =
DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
/// the node are known to be legal, but other operands of the node may need
/// legalization as well as the specified one.
bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
- DEBUG(dbgs() << "Split node operand: ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
SDValue Res = SDValue();
// See if the target wants to custom split this node.
// Use the appropriate scalar instruction on the split subvectors before
// reducing the now partially reduced smaller vector.
- SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi);
- return DAG.getNode(N->getOpcode(), dl, ResVT, Partial);
+ SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
+ return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
}
SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
N->isCompressingStore());
- MMO = DAG.getMachineFunction().
- getMachineMemOperand(N->getPointerInfo(),
- MachineMemOperand::MOStore, HiMemVT.getStoreSize(),
- SecondHalfAlignment, N->getAAInfo(), N->getRanges());
+ unsigned HiOffset = LoMemVT.getStoreSize();
+
+ MMO = DAG.getMachineFunction().getMachineMemOperand(
+ N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
+ HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(),
+ N->getRanges());
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
N->isTruncatingStore(), N->isCompressingStore());
return SplitVecOp_UnaryOp(N);
SDLoc DL(N);
- // Extract the halves of the input via extract_subvector.
+ // Get the split input vector.
SDValue InLoVec, InHiVec;
- std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL);
+ GetSplitVector(InVec, InLoVec, InHiVec);
// Truncate them to 1/2 the element size.
EVT HalfElementVT = IsFloat ?
EVT::getFloatingPointVT(InElementSize/2) :
//===----------------------------------------------------------------------===//
void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
- DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
+ dbgs() << "\n");
// See if the target wants to custom widen this node.
if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
// Widen Vector Operand
//===----------------------------------------------------------------------===//
bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
- DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
- N->dump(&DAG);
- dbgs() << "\n");
+ LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
+ dbgs() << "\n");
SDValue Res = SDValue();
// See if the target wants to custom widen this node.
}
SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
- assert(OpNo == 3 && "Can widen only data operand of mstore");
+ assert((OpNo == 2 || OpNo == 3) &&
+ "Can widen only data or mask operand of mstore");
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
SDValue Mask = MST->getMask();
EVT MaskVT = Mask.getValueType();
SDValue StVal = MST->getValue();
- // Widen the value
- SDValue WideVal = GetWidenedVector(StVal);
SDLoc dl(N);
- // The mask should be widened as well.
- EVT WideVT = WideVal.getValueType();
- EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(),
- WideVT.getVectorNumElements());
- Mask = ModifyToType(Mask, WideMaskVT, true);
+ if (OpNo == 3) {
+ // Widen the value
+ StVal = GetWidenedVector(StVal);
+
+ // The mask should be widened as well.
+ EVT WideVT = StVal.getValueType();
+ EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
+ MaskVT.getVectorElementType(),
+ WideVT.getVectorNumElements());
+ Mask = ModifyToType(Mask, WideMaskVT, true);
+ } else {
+ EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
+ Mask = ModifyToType(Mask, WideMaskVT, true);
+
+ EVT ValueVT = StVal.getValueType();
+ if (getTypeAction(ValueVT) == TargetLowering::TypeWidenVector)
+ StVal = GetWidenedVector(StVal);
+ else {
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
+ ValueVT.getVectorElementType(),
+ WideMaskVT.getVectorNumElements());
+ StVal = ModifyToType(StVal, WideVT);
+ }
+ }
assert(Mask.getValueType().getVectorNumElements() ==
- WideVal.getValueType().getVectorNumElements() &&
+ StVal.getValueType().getVectorNumElements() &&
"Mask and data vectors should have the same number of elements");
- return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(),
+ return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
Mask, MST->getMemoryVT(), MST->getMemOperand(),
false, MST->isCompressingStore());
}
OSDN Git Service
