From ff28671f85a3edcf6ded7c7c74f80a3763cd98d9 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Fri, 19 Jan 2018 20:54:18 +0000 Subject: [PATCH] [SystemZ] Directly use CC result of compare-and-swap In order to implement a test whether a compare-and-swap succeeded, the SystemZ back-end currently emits a rather inefficient sequence of first converting the CC result into an integer, and then testing that integer against zero. This commit changes the back-end to simply directly test the CC value set by the compare-and-swap instruction. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@322988 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/SystemZ/SystemZISelLowering.cpp | 124 +++++++++++++++++++++++++++++ lib/Target/SystemZ/SystemZISelLowering.h | 2 + test/CodeGen/SystemZ/cmpxchg-01.ll | 80 +++++++++++++++++++ test/CodeGen/SystemZ/cmpxchg-02.ll | 79 ++++++++++++++++++ test/CodeGen/SystemZ/cmpxchg-03.ll | 39 +++++++++ test/CodeGen/SystemZ/cmpxchg-04.ll | 38 +++++++++ test/CodeGen/SystemZ/cmpxchg-06.ll | 51 ++++++++++++ 7 files changed, 413 insertions(+) diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp index 0003c290bd7..f7fc061473a 100644 --- a/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -2161,6 +2161,24 @@ static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, C.CCMask = NewCCMask; } +// See whether the comparison argument contains a redundant AND +// and remove it if so. This sometimes happens due to the generic +// BRCOND expansion. +static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, + Comparison &C) { + if (C.Op0.getOpcode() != ISD::AND) + return; + auto *Mask = dyn_cast(C.Op0.getOperand(1)); + if (!Mask) + return; + KnownBits Known; + DAG.computeKnownBits(C.Op0.getOperand(0), Known); + if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue()) + return; + + C.Op0 = C.Op0.getOperand(0); +} + // Return a Comparison that tests the condition-code result of intrinsic // node Call against constant integer CC using comparison code Cond. // Opcode is the opcode of the SystemZISD operation for the intrinsic @@ -2235,6 +2253,7 @@ static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, else C.ICmpType = SystemZICMP::SignedOnly; C.CCMask &= ~SystemZ::CCMASK_CMP_UO; + adjustForRedundantAnd(DAG, DL, C); adjustZeroCmp(DAG, DL, C); adjustSubwordCmp(DAG, DL, C); adjustForSubtraction(DAG, DL, C); @@ -5409,6 +5428,109 @@ SDValue SystemZTargetLowering::combineSHIFTROT( return SDValue(); } +static bool combineCCMask(SDValue &Glue, int &CCValid, int &CCMask) { + // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code + // set by the glued instruction using the CCValid / CCMask masks, + // If the glued instruction is itself a (ICMP (SELECT_CCMASK)) testing + // the condition code set by some other instruction, see whether we + // can directly use that condition code. + bool Invert = false; + + // Verify that we have an appropriate mask for a EQ or NE comparison. + if (CCValid != SystemZ::CCMASK_ICMP) + return false; + if (CCMask == SystemZ::CCMASK_CMP_NE) + Invert = !Invert; + else if (CCMask != SystemZ::CCMASK_CMP_EQ) + return false; + + // Verify that we have an ICMP that is the single user of a SELECT_CCMASK. + SDNode *ICmp = Glue.getNode(); + if (ICmp->getOpcode() != SystemZISD::ICMP) + return false; + SDNode *Select = ICmp->getOperand(0).getNode(); + if (Select->getOpcode() != SystemZISD::SELECT_CCMASK) + return false; + if (!Select->hasOneUse()) + return false; + + // Verify that the ICMP compares against one of select values. + auto *CompareVal = dyn_cast(ICmp->getOperand(1)); + if (!CompareVal) + return false; + auto *TrueVal = dyn_cast(Select->getOperand(0)); + if (!TrueVal) + return false; + auto *FalseVal = dyn_cast(Select->getOperand(1)); + if (!FalseVal) + return false; + if (CompareVal->getZExtValue() == FalseVal->getZExtValue()) + Invert = !Invert; + else if (CompareVal->getZExtValue() != TrueVal->getZExtValue()) + return false; + + // Compute the effective CC mask for the new branch or select. + auto *NewCCValid = dyn_cast(Select->getOperand(2)); + auto *NewCCMask = dyn_cast(Select->getOperand(3)); + if (!NewCCValid || !NewCCMask) + return false; + CCValid = NewCCValid->getZExtValue(); + CCMask = NewCCMask->getZExtValue(); + if (Invert) + CCMask ^= CCValid; + + // Return the updated Glue link. + Glue = Select->getOperand(4); + return true; +} + +SDValue SystemZTargetLowering::combineBR_CCMASK( + SDNode *N, DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + + // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK. + auto *CCValid = dyn_cast(N->getOperand(1)); + auto *CCMask = dyn_cast(N->getOperand(2)); + if (!CCValid || !CCMask) + return SDValue(); + + int CCValidVal = CCValid->getZExtValue(); + int CCMaskVal = CCMask->getZExtValue(); + SDValue Glue = N->getOperand(4); + + if (combineCCMask(Glue, CCValidVal, CCMaskVal)) + return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0), + N->getOperand(0), + DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32), + DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32), + N->getOperand(3), Glue); + return SDValue(); +} + +SDValue SystemZTargetLowering::combineSELECT_CCMASK( + SDNode *N, DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + + // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK. + auto *CCValid = dyn_cast(N->getOperand(2)); + auto *CCMask = dyn_cast(N->getOperand(3)); + if (!CCValid || !CCMask) + return SDValue(); + + int CCValidVal = CCValid->getZExtValue(); + int CCMaskVal = CCMask->getZExtValue(); + SDValue Glue = N->getOperand(4); + + if (combineCCMask(Glue, CCValidVal, CCMaskVal)) + return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0), + N->getOperand(0), + N->getOperand(1), + DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32), + DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32), + Glue); + return SDValue(); +} + SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { switch(N->getOpcode()) { @@ -5427,6 +5549,8 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, case ISD::SRA: case ISD::SRL: case ISD::ROTL: return combineSHIFTROT(N, DCI); + case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI); + case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI); } return SDValue(); diff --git a/lib/Target/SystemZ/SystemZISelLowering.h b/lib/Target/SystemZ/SystemZISelLowering.h index a74ca69c08b..be20cd61996 100644 --- a/lib/Target/SystemZ/SystemZISelLowering.h +++ b/lib/Target/SystemZ/SystemZISelLowering.h @@ -581,6 +581,8 @@ private: SDValue combineFP_ROUND(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineBSWAP(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineSHIFTROT(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue combineBR_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue combineSELECT_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const; // If the last instruction before MBBI in MBB was some form of COMPARE, // try to replace it with a COMPARE AND BRANCH just before MBBI. diff --git a/test/CodeGen/SystemZ/cmpxchg-01.ll b/test/CodeGen/SystemZ/cmpxchg-01.ll index b3084ad1b54..82c67811fb9 100644 --- a/test/CodeGen/SystemZ/cmpxchg-01.ll +++ b/test/CodeGen/SystemZ/cmpxchg-01.ll @@ -89,3 +89,83 @@ define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) { ret i32 %res } + +declare void @g() + +; Check using the comparison result for a branch. +; CHECK-LABEL: f4 +; CHECK-MAIN-LABEL: f4: +; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} +; CHECK-MAIN-DAG: sll %r2, 3 +; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN: [[LOOP:\.[^ ]*]]: +; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2) +; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0 +; CHECK-MAIN: cr [[TMP]], %r3 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 +; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}}) +; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) +; CHECK-MAIN: jl [[LOOP]] +; CHECK-MAIN: [[EXIT]]: +; CHECK-MAIN-NEXT: jlh [[LABEL:\.[^ ]*]] +; CHECK-MAIN: jg g +; CHECK-MAIN: [[LABEL]]: +; CHECK-MAIN: br %r14 +; +; CHECK-SHIFT-LABEL: f4: +; CHECK-SHIFT: sll %r2, 3 +; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], %r2 +; CHECK-SHIFT: rll +; CHECK-SHIFT: rll {{%r[0-9]+}}, %r4, -8([[NEGSHIFT]]) +define void @f4(i8 *%src, i8 %cmp, i8 %swap) { + %pair = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst + %cond = extractvalue { i8, i1 } %pair, 1 + br i1 %cond, label %call, label %exit + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + +; ... and the same with the inverted direction. +; CHECK-MAIN-LABEL: f5: +; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} +; CHECK-MAIN-DAG: sll %r2, 3 +; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN: [[LOOP:\.[^ ]*]]: +; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2) +; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0 +; CHECK-MAIN: cr [[TMP]], %r3 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 +; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}}) +; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) +; CHECK-MAIN: jl [[LOOP]] +; CHECK-MAIN: [[EXIT]]: +; CHECK-MAIN-NEXT: jlh [[LABEL:\.[^ ]*]] +; CHECK-MAIN: br %r14 +; CHECK-MAIN: [[LABEL]]: +; CHECK-MAIN: jg g +; +; CHECK-SHIFT-LABEL: f5: +; CHECK-SHIFT: sll %r2, 3 +; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], %r2 +; CHECK-SHIFT: rll +; CHECK-SHIFT: rll {{%r[0-9]+}}, %r4, -8([[NEGSHIFT]]) +define void @f5(i8 *%src, i8 %cmp, i8 %swap) { + %pair = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst + %cond = extractvalue { i8, i1 } %pair, 1 + br i1 %cond, label %exit, label %call + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + diff --git a/test/CodeGen/SystemZ/cmpxchg-02.ll b/test/CodeGen/SystemZ/cmpxchg-02.ll index e2ca7f42583..6e266a1308c 100644 --- a/test/CodeGen/SystemZ/cmpxchg-02.ll +++ b/test/CodeGen/SystemZ/cmpxchg-02.ll @@ -89,3 +89,82 @@ define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) { ret i32 %res } +declare void @g() + +; Check using the comparison result for a branch. +; CHECK-LABEL: f4 +; CHECK-MAIN-LABEL: f4: +; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} +; CHECK-MAIN-DAG: sll %r2, 3 +; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN: [[LOOP:\.[^ ]*]]: +; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2) +; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0 +; CHECK-MAIN: cr [[TMP]], %r3 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 +; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}}) +; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) +; CHECK-MAIN: jl [[LOOP]] +; CHECK-MAIN: [[EXIT]]: +; CHECK-MAIN-NEXT: jlh [[LABEL:\.[^ ]*]] +; CHECK-MAIN: jg g +; CHECK-MAIN: [[LABEL]]: +; CHECK-MAIN: br %r14 +; +; CHECK-SHIFT-LABEL: f4: +; CHECK-SHIFT: sll %r2, 3 +; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], %r2 +; CHECK-SHIFT: rll +; CHECK-SHIFT: rll {{%r[0-9]+}}, %r4, -16([[NEGSHIFT]]) +define void @f4(i16 *%src, i16 %cmp, i16 %swap) { + %pair = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst + %cond = extractvalue { i16, i1 } %pair, 1 + br i1 %cond, label %call, label %exit + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + +; ... and the same with the inverted direction. +; CHECK-MAIN-LABEL: f5: +; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} +; CHECK-MAIN-DAG: sll %r2, 3 +; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN: [[LOOP:\.[^ ]*]]: +; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2) +; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0 +; CHECK-MAIN: cr [[TMP]], %r3 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 +; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}}) +; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) +; CHECK-MAIN: jl [[LOOP]] +; CHECK-MAIN: [[EXIT]]: +; CHECK-MAIN-NEXT: jlh [[LABEL:\.[^ ]*]] +; CHECK-MAIN: br %r14 +; CHECK-MAIN: [[LABEL]]: +; CHECK-MAIN: jg g +; +; CHECK-SHIFT-LABEL: f5: +; CHECK-SHIFT: sll %r2, 3 +; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], %r2 +; CHECK-SHIFT: rll +; CHECK-SHIFT: rll {{%r[0-9]+}}, %r4, -16([[NEGSHIFT]]) +define void @f5(i16 *%src, i16 %cmp, i16 %swap) { + %pair = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst + %cond = extractvalue { i16, i1 } %pair, 1 + br i1 %cond, label %exit, label %call + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + diff --git a/test/CodeGen/SystemZ/cmpxchg-03.ll b/test/CodeGen/SystemZ/cmpxchg-03.ll index d2576e4291b..153e4400acc 100644 --- a/test/CodeGen/SystemZ/cmpxchg-03.ll +++ b/test/CodeGen/SystemZ/cmpxchg-03.ll @@ -155,3 +155,42 @@ define i32 @f13(i32 %cmp, i32 %swap, i32 *%src) { %res = zext i1 %val to i32 ret i32 %res } + +declare void @g() + +; Check using the comparison result for a branch. +; CHECK-LABEL: f14 +; CHECK: cs %r2, %r3, 0(%r4) +; CHECK-NEXT: jge g +; CHECK: br %r14 +define void @f14(i32 %cmp, i32 %swap, i32 *%src) { + %pairval = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst + %cond = extractvalue { i32, i1 } %pairval, 1 + br i1 %cond, label %call, label %exit + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + +; ... and the same with the inverted direction. +; CHECK-LABEL: f15 +; CHECK: cs %r2, %r3, 0(%r4) +; CHECK-NEXT: jgl g +; CHECK: br %r14 +define void @f15(i32 %cmp, i32 %swap, i32 *%src) { + %pairval = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst + %cond = extractvalue { i32, i1 } %pairval, 1 + br i1 %cond, label %exit, label %call + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + diff --git a/test/CodeGen/SystemZ/cmpxchg-04.ll b/test/CodeGen/SystemZ/cmpxchg-04.ll index f461315b999..442a7ed0ae6 100644 --- a/test/CodeGen/SystemZ/cmpxchg-04.ll +++ b/test/CodeGen/SystemZ/cmpxchg-04.ll @@ -120,3 +120,41 @@ define i32 @f10(i64 %cmp, i64 %swap, i64 *%src) { ret i32 %res } +declare void @g() + +; Check using the comparison result for a branch. +; CHECK-LABEL: f11 +; CHECK: csg %r2, %r3, 0(%r4) +; CHECK-NEXT: jge g +; CHECK: br %r14 +define void @f11(i64 %cmp, i64 %swap, i64 *%src) { + %pairval = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst + %cond = extractvalue { i64, i1 } %pairval, 1 + br i1 %cond, label %call, label %exit + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + +; ... and the same with the inverted direction. +; CHECK-LABEL: f12 +; CHECK: csg %r2, %r3, 0(%r4) +; CHECK-NEXT: jgl g +; CHECK: br %r14 +define void @f12(i64 %cmp, i64 %swap, i64 *%src) { + %pairval = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst + %cond = extractvalue { i64, i1 } %pairval, 1 + br i1 %cond, label %exit, label %call + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + diff --git a/test/CodeGen/SystemZ/cmpxchg-06.ll b/test/CodeGen/SystemZ/cmpxchg-06.ll index 7da2ea0fde8..b3168e2a6b0 100644 --- a/test/CodeGen/SystemZ/cmpxchg-06.ll +++ b/test/CodeGen/SystemZ/cmpxchg-06.ll @@ -129,3 +129,54 @@ define i32 @f10(i128 %cmp, i128 %swap, i128 *%src) { %res = zext i1 %val to i32 ret i32 %res } + +declare void @g() + +; Check using the comparison result for a branch. +; CHECK-LABEL: f11 +; CHECK-DAG: lg %r1, 8(%r3) +; CHECK-DAG: lg %r0, 0(%r3) +; CHECK-DAG: lg %r13, 8(%r2) +; CHECK-DAG: lg %r12, 0(%r2) +; CHECK: cdsg %r12, %r0, 0(%r4) +; CHECK-NEXT: jl [[LABEL:\.[^ ]*]] +; CHECK: jg g +; CHECK: [[LABEL]]: +; CHECK: br %r14 +define void @f11(i128 %cmp, i128 %swap, i128 *%src) { + %pairval = cmpxchg i128 *%src, i128 %cmp, i128 %swap seq_cst seq_cst + %cond = extractvalue { i128, i1 } %pairval, 1 + br i1 %cond, label %call, label %exit + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + +; ... and the same with the inverted direction. +; CHECK-LABEL: f12 +; CHECK-DAG: lg %r1, 8(%r3) +; CHECK-DAG: lg %r0, 0(%r3) +; CHECK-DAG: lg %r13, 8(%r2) +; CHECK-DAG: lg %r12, 0(%r2) +; CHECK: cdsg %r12, %r0, 0(%r4) +; CHECK-NEXT: jl [[LABEL:\.[^ ]*]] +; CHECK: br %r14 +; CHECK: [[LABEL]]: +; CHECK: jg g +define void @f12(i128 %cmp, i128 %swap, i128 *%src) { + %pairval = cmpxchg i128 *%src, i128 %cmp, i128 %swap seq_cst seq_cst + %cond = extractvalue { i128, i1 } %pairval, 1 + br i1 %cond, label %exit, label %call + +call: + tail call void @g() + br label %exit + +exit: + ret void +} + -- 2.11.0