From f68617b74f553e2e9a6646bb20d798215306934a Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 18 Jan 2019 20:14:46 +0000 Subject: [PATCH] [X86] Lower avx512f scatter intrinsics to X86MaskedScatterSDNode instead of going directly to MachineSDNode. This sends these intrinsics through isel in a much more normal way. This should allow addressing mode matching in isel to make better use of the displacement field. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@351583 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/X86/X86ISelLowering.cpp | 22 +++++++-- lib/Target/X86/X86IntrinsicsInfo.h | 96 +++++++++++++++++++------------------- 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index d4179993f02..5150dec9432 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -4822,6 +4822,17 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags |= MachineMemOperand::MOLoad; break; } + case SCATTER: { + Info.ptrVal = nullptr; + MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType()); + MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType()); + unsigned NumElts = std::min(DataVT.getVectorNumElements(), + IndexVT.getVectorNumElements()); + Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts); + Info.align = 1; + Info.flags |= MachineMemOperand::MOStore; + break; + } default: return false; } @@ -22449,8 +22460,6 @@ static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, if (!C) return SDValue(); SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); - SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); - SDValue Segment = DAG.getRegister(0, MVT::i32); unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(), Src.getSimpleValueType().getVectorNumElements()); MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts); @@ -22460,10 +22469,13 @@ static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, if (Mask.getValueType() != MaskVT) Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); + MemIntrinsicSDNode *MemIntr = cast(Op); + SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); - SDValue Ops[] = {Base, Scale, Index, Disp, Segment, Mask, Src, Chain}; - SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops); - return SDValue(Res, 1); + SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale}; + SDValue Res = DAG.getTargetMemSDNode( + VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand()); + return Res.getValue(1); } static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, diff --git a/lib/Target/X86/X86IntrinsicsInfo.h b/lib/Target/X86/X86IntrinsicsInfo.h index 46f71609e82..1604c690a19 100644 --- a/lib/Target/X86/X86IntrinsicsInfo.h +++ b/lib/Target/X86/X86IntrinsicsInfo.h @@ -249,47 +249,47 @@ static const IntrinsicData IntrinsicsWithChain[] = { X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_mem_512, TRUNCATE_TO_MEM_VI8, X86ISD::VTRUNCUS, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_dpd_512, SCATTER, X86::VSCATTERDPDZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_dpi_512, SCATTER, X86::VPSCATTERDDZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_dpq_512, SCATTER, X86::VPSCATTERDQZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_dps_512, SCATTER, X86::VSCATTERDPSZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_qpd_512, SCATTER, X86::VSCATTERQPDZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_qpi_512, SCATTER, X86::VPSCATTERQDZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_qpq_512, SCATTER, X86::VPSCATTERQQZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatter_qps_512, SCATTER, X86::VSCATTERQPSZmr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_df, SCATTER, X86::VSCATTERQPDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_di, SCATTER, X86::VPSCATTERQQZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_df, SCATTER, X86::VSCATTERQPDZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_di, SCATTER, X86::VPSCATTERQQZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_sf, SCATTER, X86::VSCATTERQPSZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_si, SCATTER, X86::VPSCATTERQDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_sf, SCATTER, X86::VSCATTERQPSZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_si, SCATTER, X86::VPSCATTERQDZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv2_df, SCATTER, X86::VSCATTERDPDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv2_di, SCATTER, X86::VPSCATTERDQZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv4_df, SCATTER, X86::VSCATTERDPDZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv4_di, SCATTER, X86::VPSCATTERDQZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv4_sf, SCATTER, X86::VSCATTERDPSZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv4_si, SCATTER, X86::VPSCATTERDDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv8_sf, SCATTER, X86::VSCATTERDPSZ256mr, 0), - X86_INTRINSIC_DATA(avx512_mask_scattersiv8_si, SCATTER, X86::VPSCATTERDDZ256mr, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_dpd_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_dpi_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_dpq_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_dps_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_qpd_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_qpi_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_qpq_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatter_qps_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_si, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_si, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv2_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv2_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv4_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv4_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv4_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv4_si, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv8_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_mask_scattersiv8_si, SCATTER, 0, 0), - X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, X86::VSCATTERDPDZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, X86::VPSCATTERDDZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, X86::VPSCATTERDQZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_dps_512, SCATTER, X86::VSCATTERDPSZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_qpd_512, SCATTER, X86::VSCATTERQPDZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_qpi_512, SCATTER, X86::VPSCATTERQDZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_qpq_512, SCATTER, X86::VPSCATTERQQZmr, 0), - X86_INTRINSIC_DATA(avx512_scatter_qps_512, SCATTER, X86::VSCATTERQPSZmr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv2_df, SCATTER, X86::VSCATTERQPDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv2_di, SCATTER, X86::VPSCATTERQQZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv4_df, SCATTER, X86::VSCATTERQPDZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv4_di, SCATTER, X86::VPSCATTERQQZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv4_sf, SCATTER, X86::VSCATTERQPSZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv4_si, SCATTER, X86::VPSCATTERQDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv8_sf, SCATTER, X86::VSCATTERQPSZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scatterdiv8_si, SCATTER, X86::VPSCATTERQDZ256mr, 0), + X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_dps_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_qpd_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_qpi_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_qpq_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatter_qps_512, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv2_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv2_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv4_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv4_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv4_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv4_si, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv8_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scatterdiv8_si, SCATTER, 0, 0), X86_INTRINSIC_DATA(avx512_scatterpf_dpd_512, PREFETCH, X86::VSCATTERPF0DPDm, X86::VSCATTERPF1DPDm), X86_INTRINSIC_DATA(avx512_scatterpf_dps_512, PREFETCH, X86::VSCATTERPF0DPSm, @@ -298,14 +298,14 @@ static const IntrinsicData IntrinsicsWithChain[] = { X86::VSCATTERPF1QPDm), X86_INTRINSIC_DATA(avx512_scatterpf_qps_512, PREFETCH, X86::VSCATTERPF0QPSm, X86::VSCATTERPF1QPSm), - X86_INTRINSIC_DATA(avx512_scattersiv2_df, SCATTER, X86::VSCATTERDPDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv2_di, SCATTER, X86::VPSCATTERDQZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv4_df, SCATTER, X86::VSCATTERDPDZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv4_di, SCATTER, X86::VPSCATTERDQZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv4_sf, SCATTER, X86::VSCATTERDPSZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv4_si, SCATTER, X86::VPSCATTERDDZ128mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv8_sf, SCATTER, X86::VSCATTERDPSZ256mr, 0), - X86_INTRINSIC_DATA(avx512_scattersiv8_si, SCATTER, X86::VPSCATTERDDZ256mr, 0), + X86_INTRINSIC_DATA(avx512_scattersiv2_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv2_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv4_df, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv4_di, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv4_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv4_si, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv8_sf, SCATTER, 0, 0), + X86_INTRINSIC_DATA(avx512_scattersiv8_si, SCATTER, 0, 0), X86_INTRINSIC_DATA(rdpmc, RDPMC, X86ISD::RDPMC_DAG, 0), X86_INTRINSIC_DATA(rdrand_16, RDRAND, X86ISD::RDRAND, 0), X86_INTRINSIC_DATA(rdrand_32, RDRAND, X86ISD::RDRAND, 0), -- 2.11.0