From 390bcc0ac8c73dd81e1929382f2af5927d3d5ae7 Mon Sep 17 00:00:00 2001 From: James Molloy Date: Wed, 2 Sep 2015 10:14:54 +0000 Subject: [PATCH] [LV] Switch to using canonical induction variables. Vectorized loops only ever have one induction variable. All induction PHIs from the scalar loop are rewritten to be in terms of this single indvar. We were trying very hard to pick an indvar that already existed, even if that indvar wasn't canonical (didn't start at zero). But trying so hard is really fruitless - creating a new, canonical, indvar only results in one extra add in the worst case and that add is trivially easy to push through the PHI out of the loop by instcombine. If we try and be less clever here and instead let instcombine clean up our mess (as we do in many other places in LV), we can remove unneeded complexity. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@246630 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/LoopVectorize.cpp | 22 ++++++++-------------- test/Transforms/LoopVectorize/induction.ll | 6 ++---- test/Transforms/LoopVectorize/reverse_induction.ll | 9 +++------ 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index eaa3ecd48e4..70199ace35c 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2652,15 +2652,8 @@ void InnerLoopVectorizer::createEmptyLoop() { ConstantInt::get(ExitCountValue->getType(), VF * UF), "min.iters.check", VectorPH->getTerminator()); - // The loop index does not have to start at Zero. Find the original start - // value from the induction PHI node. If we don't have an induction variable - // then we know that it starts at zero. Builder.SetInsertPoint(VectorPH->getTerminator()); - Value *StartIdx = ExtendedIdx = - OldInduction - ? Builder.CreateZExt(OldInduction->getIncomingValueForBlock(VectorPH), - IdxTy) - : ConstantInt::get(IdxTy, 0); + Value *StartIdx = ExtendedIdx = ConstantInt::get(IdxTy, 0); // Count holds the overall loop count (N). Value *Count = Exp.expandCodeFor(ExitCount, ExitCount->getType(), @@ -3542,10 +3535,8 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, } else { // Handle other induction variables that are now based on the // canonical one. - Value *NormalizedIdx = Builder.CreateSub(Induction, ExtendedIdx, - "normalized.idx"); - NormalizedIdx = Builder.CreateSExtOrTrunc(NormalizedIdx, PhiTy); - Broadcasted = II.transform(Builder, NormalizedIdx); + auto *V = Builder.CreateSExtOrTrunc(Induction, PhiTy); + Broadcasted = II.transform(Builder, V); Broadcasted->setName("offset.idx"); } Broadcasted = getBroadcastInstrs(Broadcasted); @@ -4134,10 +4125,13 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { // Int inductions are special because we only allow one IV. if (ID.getKind() == InductionDescriptor::IK_IntInduction && - ID.getStepValue()->isOne()) { + ID.getStepValue()->isOne() && + isa(ID.getStartValue()) && + cast(ID.getStartValue())->isNullValue()) { // Use the phi node with the widest type as induction. Use the last // one if there are multiple (no good reason for doing this other - // than it is expedient). + // than it is expedient). We've checked that it begins at zero and + // steps by one, so this is a canonical induction variable. if (!Induction || PhiTy == WidestIndTy) Induction = Phi; } diff --git a/test/Transforms/LoopVectorize/induction.ll b/test/Transforms/LoopVectorize/induction.ll index 48566ef92f7..f8017dd1106 100644 --- a/test/Transforms/LoopVectorize/induction.ll +++ b/test/Transforms/LoopVectorize/induction.ll @@ -6,8 +6,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-LABEL: @multi_int_induction( ; CHECK: vector.body: ; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %normalized.idx = sub i64 %index, 0 -; CHECK: %[[VAR:.*]] = trunc i64 %normalized.idx to i32 +; CHECK: %[[VAR:.*]] = trunc i64 %index to i32 ; CHECK: %offset.idx = add i32 190, %[[VAR]] define void @multi_int_induction(i32* %A, i32 %N) { for.body.lr.ph: @@ -142,11 +141,10 @@ define i32 @max_i32_backedgetaken() nounwind readnone ssp uwtable { ; CHECK-LABEL: testoverflowcheck ; CHECK: entry ; CHECK: %[[LOAD:.*]] = load i8 -; CHECK: %[[VAL:.*]] = zext i8 %[[LOAD]] to i32 ; CHECK: br ; CHECK: scalar.ph -; CHECK: phi i32 [ %{{.*}}, %middle.block ], [ %[[VAL]], %entry ] +; CHECK: phi i8 [ %{{.*}}, %middle.block ], [ %[[LOAD]], %entry ] @e = global i8 1, align 1 @d = common global i32 0, align 4 diff --git a/test/Transforms/LoopVectorize/reverse_induction.ll b/test/Transforms/LoopVectorize/reverse_induction.ll index 6b63a0d8db6..88dd2e4d66c 100644 --- a/test/Transforms/LoopVectorize/reverse_induction.ll +++ b/test/Transforms/LoopVectorize/reverse_induction.ll @@ -96,8 +96,7 @@ loopend: ; CHECK-LABEL: @reverse_forward_induction_i64_i8( ; CHECK: vector.body ; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %normalized.idx = sub i64 %index, 0 -; CHECK: %offset.idx = sub i64 1023, %normalized.idx +; CHECK: %offset.idx = sub i64 1023, %index ; CHECK: trunc i64 %index to i8 define void @reverse_forward_induction_i64_i8() { @@ -122,10 +121,8 @@ while.end: ; CHECK-LABEL: @reverse_forward_induction_i64_i8_signed( ; CHECK: vector.body: -; CHECK: %index = phi i64 [ 129, %vector.ph ], [ %index.next, %vector.body ] -; CHECK: %normalized.idx = sub i64 %index, 129 -; CHECK: %offset.idx = sub i64 1023, %normalized.idx -; CHECK: trunc i64 %index to i8 +; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK: %offset.idx = sub i64 1023, %index define void @reverse_forward_induction_i64_i8_signed() { entry: -- 2.11.0