From f32a3fd4afb5802a6e1dd7d11b8be231c53d49d4 Mon Sep 17 00:00:00 2001 From: Matthew Simpson Date: Mon, 13 Feb 2017 16:48:00 +0000 Subject: [PATCH] [LV] Extend trunc optimization to all IVs with constant integer steps This patch extends the optimization of truncations whose operand is an induction variable with a constant integer step. Previously we were only applying this optimization to the primary induction variable. However, the cost model assumes the optimization is applied to the truncation of all integer induction variables (even regardless of step type). The transformation is now applied to the other induction variables, and I've updated the cost model to ensure it is better in sync with the transformation we actually perform. Differential Revision: https://reviews.llvm.org/D29847 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@294967 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/LoopVectorize.cpp | 32 +++++++++++++++++---------- test/Transforms/LoopVectorize/induction.ll | 31 ++++++++++++++++++++++++++ test/Transforms/LoopVectorize/reverse_iter.ll | 5 +++-- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index 401b942d307..5513c7fd62a 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4879,12 +4879,15 @@ void InnerLoopVectorizer::vectorizeBlockInLoop(BasicBlock *BB, PhiVector *PV) { // induction variable. Notice that we can only optimize the 'trunc' case // because (a) FP conversions lose precision, (b) sext/zext may wrap, and // (c) other casts depend on pointer size. - auto ID = Legal->getInductionVars()->lookup(OldInduction); - if (isa(CI) && CI->getOperand(0) == OldInduction && - ID.getConstIntStepValue()) { - widenIntInduction(OldInduction, cast(CI)); - break; - } + if (auto *Trunc = dyn_cast(CI)) + if (auto *Phi = dyn_cast(Trunc->getOperand(0))) { + auto II = Legal->getInductionVars()->find(Phi); + if (II != Legal->getInductionVars()->end()) + if (II->second.getConstIntStepValue()) { + widenIntInduction(Phi, Trunc); + break; + } + } /// Vectorize casts. Type *DestTy = @@ -7224,12 +7227,17 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I, case Instruction::Trunc: case Instruction::FPTrunc: case Instruction::BitCast: { - // We optimize the truncation of induction variable. - // The cost of these is the same as the scalar operation. - if (I->getOpcode() == Instruction::Trunc && - Legal->isInductionVariable(I->getOperand(0))) - return TTI.getCastInstrCost(I->getOpcode(), I->getType(), - I->getOperand(0)->getType()); + // We optimize the truncation of induction variables having constant + // integer steps. The cost of these truncations is the same as the scalar + // operation. + if (auto *Trunc = dyn_cast(I)) + if (auto *Phi = dyn_cast(Trunc->getOperand(0))) { + auto II = Legal->getInductionVars()->find(Phi); + if (II != Legal->getInductionVars()->end()) + if (II->second.getConstIntStepValue()) + return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(), + Trunc->getSrcTy()); + } Type *SrcScalarTy = I->getOperand(0)->getType(); Type *SrcVecTy = ToVectorTy(SrcScalarTy, VF); diff --git a/test/Transforms/LoopVectorize/induction.ll b/test/Transforms/LoopVectorize/induction.ll index 85f00b4dfc8..da20a2a7c4d 100644 --- a/test/Transforms/LoopVectorize/induction.ll +++ b/test/Transforms/LoopVectorize/induction.ll @@ -773,3 +773,34 @@ for.body: exit: ret void } + +; CHECK-LABEL: @non_primary_iv_trunc( +; CHECK: vector.body: +; CHECK-NEXT: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK: [[VEC_IND:%.*]] = phi <2 x i32> [ , %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ] +; CHECK: [[TMP3:%.*]] = add i64 %index, 0 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* %a, i64 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, i32* [[TMP4]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <2 x i32>* +; CHECK-NEXT: store <2 x i32> [[VEC_IND]], <2 x i32>* [[TMP6]], align 4 +; CHECK-NEXT: %index.next = add i64 %index, 2 +; CHECK: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], +; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body +define void @non_primary_iv_trunc(i32* %a, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] + %j = phi i64 [ %j.next, %for.body ], [ 0, %entry ] + %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i + %tmp1 = trunc i64 %j to i32 + store i32 %tmp1, i32* %tmp0, align 4 + %i.next = add nuw nsw i64 %i, 1 + %j.next = add nuw nsw i64 %j, 2 + %cond = icmp slt i64 %i.next, %n + br i1 %cond, label %for.body, label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/reverse_iter.ll b/test/Transforms/LoopVectorize/reverse_iter.ll index a6e2abda36d..bd057698280 100644 --- a/test/Transforms/LoopVectorize/reverse_iter.ll +++ b/test/Transforms/LoopVectorize/reverse_iter.ll @@ -2,7 +2,8 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" -; Make sure that the reverse iterators are calculated using 64bit arithmetic, not 32. +; PR15882: This test ensures that we do not produce wrapping arithmetic when +; creating constant reverse step vectors. ; ; int foo(int n, int *A) { ; int sum; @@ -13,7 +14,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; ;CHECK-LABEL: @foo( -;CHECK: +;CHECK: ;CHECK: ret define i32 @foo(i32 %n, i32* nocapture %A) { %1 = icmp sgt i32 %n, 0 -- 2.11.0