VPTransformState &State) {
IRBuilderBase &Builder = State.Builder;
// We shouldn't have to build scalar steps if we aren't vectorizing.
- assert(State.VF.isVector() && "VF should be greater than one");
// Get the value type and ensure it and the step have the same integer type.
Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
assert(ScalarIVTy == Step->getType() &&
};
Value *ScalarIV = CreateScalarIV(Step);
- if (State.VF.isVector()) {
- buildScalarSteps(ScalarIV, Step, IndDesc, this, State);
- return;
- }
-
- for (unsigned Part = 0; Part < State.UF; ++Part) {
- assert(!State.VF.isScalable() && "scalable vectors not yet supported.");
- Value *EntryPart;
- if (Step->getType()->isFloatingPointTy()) {
- Value *StartIdx =
- getRuntimeVFAsFloat(State.Builder, Step->getType(), State.VF * Part);
- // Floating-point operations inherit FMF via the builder's flags.
- Value *MulOp = State.Builder.CreateFMul(StartIdx, Step);
- EntryPart = State.Builder.CreateBinOp(IndDesc.getInductionOpcode(),
- ScalarIV, MulOp);
- } else {
- Value *StartIdx =
- getRuntimeVF(State.Builder, Step->getType(), State.VF * Part);
- EntryPart = State.Builder.CreateAdd(
- ScalarIV, State.Builder.CreateMul(StartIdx, Step), "induction");
- }
- State.set(this, EntryPart, Part);
- }
+ buildScalarSteps(ScalarIV, Step, IndDesc, this, State);
}
void VPInterleaveRecipe::execute(VPTransformState &State) {
; CHECK-NEXT: store i32 {{%.*}}, i32* {{%.*}}
; CHECK-NEXT: br label %pred.store.continue
; CHECK: pred.store.continue:
-; CHECK-NEXT: br i1 {{%.*}}, label %pred.store.if2, label %pred.store.continue3
-; CHECK: pred.store.if2:
+; CHECK-NEXT: br i1 {{%.*}}, label %pred.store.if1, label %pred.store.continue2
+; CHECK: pred.store.if1:
; CHECK-NEXT: store i32 {{%.*}}, i32* {{%.*}}
-; CHECK-NEXT: br label %pred.store.continue3
-; CHECK: pred.store.continue3:
+; CHECK-NEXT: br label %pred.store.continue2
+; CHECK: pred.store.continue2:
entry:
br label %while.body
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF4:%.*]], label [[PRED_UDIV_CONTINUE5]]
-; UNROLL-NO-VF: pred.udiv.if4:
+; UNROLL-NO-VF: pred.udiv.if3:
; UNROLL-NO-VF-NEXT: [[INDUCTION2:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION2]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE5]]
-; UNROLL-NO-VF: pred.udiv.continue5:
+; UNROLL-NO-VF: pred.udiv.continue4:
; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF4]] ]
; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI1]], [[TMP5]]
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; UNROLL-NO-VF: pred.udiv.if7:
+; UNROLL-NO-VF: pred.udiv.if4:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[INDUCTION5]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; UNROLL-NO-VF: pred.udiv.continue8:
+; UNROLL-NO-VF: pred.udiv.continue5:
; UNROLL-NO-VF-NEXT: [[TMP7]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP6]], [[PRED_UDIV_IF7]] ]
; UNROLL-NO-VF-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP9]] = add i32 [[VEC_PHI2]], [[TMP5]]
; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NO-VF: pred.store.continue:
; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]]
-; UNROLL-NO-VF: pred.store.if9:
+; UNROLL-NO-VF: pred.store.if6:
; UNROLL-NO-VF-NEXT: [[INDUCTION3:%.*]] = add i32 [[INDEX]], 1
; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 [[INDUCTION3]]
; UNROLL-NO-VF-NEXT: store i32 [[INDUCTION5]], i32* [[TMP11]], align 4
; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; UNROLL-NO-VF: pred.store.continue10:
+; UNROLL-NO-VF: pred.store.continue7:
; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP8]], i32 [[VEC_PHI]]
; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP9]], i32 [[VEC_PHI2]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; VEC1_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE]]
; VEC1_INTERL2: pred.store.continue:
; VEC1_INTERL2-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; VEC1_INTERL2: pred.store.if3:
+; VEC1_INTERL2: pred.store.if2:
; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fadd fast float [[TMP0]], 1.000000e+00
; VEC1_INTERL2-NEXT: store float [[TMP7]], float* [[TMP2]], align 4
; VEC1_INTERL2-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; VEC1_INTERL2: pred.store.continue4:
+; VEC1_INTERL2: pred.store.continue3:
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC1_INTERL2-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; UNROLL-NO-VF-NEXT: [[TMP32:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP28]], [[PRED_UREM_IF]] ]
; UNROLL-NO-VF-NEXT: [[TMP33:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP29]], [[PRED_UREM_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP25]], label [[PRED_UREM_IF32:%.*]], label [[PRED_UREM_CONTINUE33]]
-; UNROLL-NO-VF: pred.urem.if32:
+; UNROLL-NO-VF: pred.urem.if31:
; UNROLL-NO-VF-NEXT: [[TMP34:%.*]] = sdiv i32 [[TMP17]], [[TMP9]]
; UNROLL-NO-VF-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP19]], [[TMP11]]
; UNROLL-NO-VF-NEXT: [[TMP36:%.*]] = srem i32 [[TMP21]], [[TMP13]]
; UNROLL-NO-VF-NEXT: [[TMP37:%.*]] = urem i32 [[TMP23]], [[TMP15]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UREM_CONTINUE33]]
-; UNROLL-NO-VF: pred.urem.continue33:
+; UNROLL-NO-VF: pred.urem.continue32:
; UNROLL-NO-VF-NEXT: [[TMP38:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP34]], [[PRED_UREM_IF32]] ]
; UNROLL-NO-VF-NEXT: [[TMP39:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP35]], [[PRED_UREM_IF32]] ]
; UNROLL-NO-VF-NEXT: [[TMP40:%.*]] = phi i32 [ poison, [[PRED_UREM_CONTINUE]] ], [ [[TMP36]], [[PRED_UREM_IF32]] ]
; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_SDIV_IF]] ]
; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_SDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP7]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8]]
-; UNROLL-NO-VF: pred.sdiv.if7:
+; UNROLL-NO-VF: pred.sdiv.if6:
; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[BSD]], i64 [[INDUCTION6]]
; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, !alias.scope !22
; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = sdiv i32 [[TMP5]], [[TMP3]]
; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = sdiv i32 [[TMP15]], [[TMP16]]
; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE8]]
-; UNROLL-NO-VF: pred.sdiv.continue8:
+; UNROLL-NO-VF: pred.sdiv.continue7:
; UNROLL-NO-VF-NEXT: [[TMP18:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP16]], [[PRED_SDIV_IF7]] ]
; UNROLL-NO-VF-NEXT: [[TMP19:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP17]], [[PRED_SDIV_IF7]] ]
; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = xor i1 [[TMP6]], true
; UNROLL-NO-VF-NEXT: [[TMP20:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP18]], [[PRED_SDIV_IF]] ]
; UNROLL-NO-VF-NEXT: [[TMP21:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP19]], [[PRED_SDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[TMP15]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8]]
-; UNROLL-NO-VF: pred.sdiv.if7:
+; UNROLL-NO-VF: pred.sdiv.if6:
; UNROLL-NO-VF-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[BSD]], i64 [[INDUCTION6]]
; UNROLL-NO-VF-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, !alias.scope !31
; UNROLL-NO-VF-NEXT: [[TMP24:%.*]] = sdiv i32 [[TMP5]], [[TMP3]]
; UNROLL-NO-VF-NEXT: [[TMP25:%.*]] = sdiv i32 [[TMP23]], [[TMP24]]
; UNROLL-NO-VF-NEXT: br label [[PRED_SDIV_CONTINUE8]]
-; UNROLL-NO-VF: pred.sdiv.continue8:
+; UNROLL-NO-VF: pred.sdiv.continue7:
; UNROLL-NO-VF-NEXT: [[TMP26:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP24]], [[PRED_SDIV_IF7]] ]
; UNROLL-NO-VF-NEXT: [[TMP27:%.*]] = phi i32 [ poison, [[PRED_SDIV_CONTINUE]] ], [ [[TMP25]], [[PRED_SDIV_IF7]] ]
; UNROLL-NO-VF-NEXT: [[TMP28:%.*]] = xor i1 [[TMP8]], true, !dbg [[DBG34]]
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if3:
+; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add nsw i32 [[TMP3]], [[X]]
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP3]], [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue4:
+; UNROLL-NO-VF: pred.udiv.continue3:
; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = xor i1 [[C]], true
; UNROLL-NO-VF-NEXT: [[TMP11:%.*]] = xor i1 [[C]], true
; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL: pred.store.continue:
; UNROLL-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL: pred.store.if2:
+; UNROLL: pred.store.if1:
; UNROLL-NEXT: [[TMP7:%.*]] = add nsw i32 [[TMP3]], 20
; UNROLL-NEXT: store i32 [[TMP7]], i32* [[TMP1]], align 4
; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; UNROLL: pred.store.continue3:
+; UNROLL: pred.store.continue2:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; UNROLL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: pred.store.continue:
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY: pred.store.if2:
+; UNROLL-NOSIMPLIFY: pred.store.if1:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = add nsw i32 [[TMP3]], 20
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP7]], i32* [[TMP1]], align 4
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE3]]
-; UNROLL-NOSIMPLIFY: pred.store.continue3:
+; UNROLL-NOSIMPLIFY: pred.store.continue2:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UNROLL-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4
; UNROLL-NEXT: store i32 [[TMP8]], i32* [[TMP7]], align 4
; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; UNROLL: pred.store.continue4:
+; UNROLL: pred.store.continue3:
; UNROLL-NEXT: [[TMP9:%.*]] = add i32 [[VEC_PHI]], 1
; UNROLL-NEXT: [[TMP10:%.*]] = add i32 [[VEC_PHI2]], 1
; UNROLL-NEXT: [[TMP11:%.*]] = xor i1 [[COND_2]], true
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: pred.store.continue:
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[COND_2]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; UNROLL-NOSIMPLIFY: pred.store.if3:
+; UNROLL-NOSIMPLIFY: pred.store.if2:
; UNROLL-NOSIMPLIFY-NEXT: [[INDUCTION1:%.*]] = add i64 [[OFFSET_IDX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = getelementptr inbounds [768 x i32], [768 x i32]* [[PTR]], i64 0, i64 [[INDUCTION1]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP7]], i32* [[TMP6]], align 4
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE4]]
-; UNROLL-NOSIMPLIFY: pred.store.continue4:
+; UNROLL-NOSIMPLIFY: pred.store.continue3:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = add i32 [[VEC_PHI]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP9:%.*]] = add i32 [[VEC_PHI2]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP10:%.*]] = xor i1 [[COND_2]], true
; UNROLL-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; UNROLL-NEXT: store i8 [[TMP7]], i8* [[TMP4]], align 1
; UNROLL-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; UNROLL: pred.store.continue4:
+; UNROLL: pred.store.continue3:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
; UNROLL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: pred.store.continue:
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
-; UNROLL-NOSIMPLIFY: pred.store.if3:
+; UNROLL-NOSIMPLIFY: pred.store.if2:
; UNROLL-NOSIMPLIFY-NEXT: [[INDUCTION2:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = getelementptr i8, i8* undef, i64 [[INDUCTION2]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP7]], i8* [[TMP4]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; UNROLL-NOSIMPLIFY: pred.store.continue4:
+; UNROLL-NOSIMPLIFY: pred.store.continue3:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], undef
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: pred.store.continue:
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
-; UNROLL-NOSIMPLIFY: pred.store.if3:
+; UNROLL-NOSIMPLIFY: pred.store.if2:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i8, i8* [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP7]], i8* [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label [[PRED_STORE_CONTINUE6]]
-; UNROLL-NOSIMPLIFY: pred.store.continue4:
+; UNROLL-NOSIMPLIFY: pred.store.continue3:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) {
; CHECK-LABEL: @test_scalarize_with_branch_cond(
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %pred.store.continue8 ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %pred.store.continue5 ]
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i1
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i1 false, [[TMP0]]
; CHECK-NEXT: [[INDUCTION:%.*]] = add i1 [[OFFSET_IDX]], false
; CHECK-NEXT: br label %pred.store.continue
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ poison, %vector.body ], [ [[TMP4]], %pred.store.if ]
-; CHECK-NEXT: br i1 [[INDUCTION3]], label %pred.store.if7, label %pred.store.continue8
-; CHECK: pred.store.if7:
+; CHECK-NEXT: br i1 [[INDUCTION3]], label %pred.store.if4, label %pred.store.continue5
+; CHECK: pred.store.if4:
; CHECK-NEXT: [[INDUCTION5:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[INDUCTION5]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr %src, i64 [[INDUCTION5]]
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4
-; CHECK-NEXT: br label %pred.store.continue8
-; CHECK: pred.store.continue8:
-; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, %pred.store.continue ], [ [[TMP7]], %pred.store.if7 ]
+; CHECK-NEXT: br label %pred.store.continue5
+; CHECK: pred.store.continue5:
+; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, %pred.store.continue ], [ [[TMP7]], %pred.store.if4 ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP9]], label %middle.block, label %vector.body
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE]]
; VF1UF4: pred.store.continue:
; VF1UF4-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; VF1UF4: pred.store.if7:
+; VF1UF4: pred.store.if4:
; VF1UF4-NEXT: [[INDUCTION1:%.*]] = add i32 [[INDEX]], 1
; VF1UF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[INDUCTION1]]
; VF1UF4-NEXT: store i32 13, i32* [[TMP5]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; VF1UF4: pred.store.continue8:
+; VF1UF4: pred.store.continue5:
; VF1UF4-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; VF1UF4: pred.store.if9:
+; VF1UF4: pred.store.if6:
; VF1UF4-NEXT: [[INDUCTION2:%.*]] = add i32 [[INDEX]], 2
; VF1UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[INDUCTION2]]
; VF1UF4-NEXT: store i32 13, i32* [[TMP6]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; VF1UF4: pred.store.continue10:
+; VF1UF4: pred.store.continue7:
; VF1UF4-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; VF1UF4: pred.store.if11:
+; VF1UF4: pred.store.if8:
; VF1UF4-NEXT: [[INDUCTION3:%.*]] = add i32 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[INDUCTION3]]
; VF1UF4-NEXT: store i32 13, i32* [[TMP7]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; VF1UF4: pred.store.continue12:
+; VF1UF4: pred.store.continue9:
; VF1UF4-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; VF1UF4-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
; VF1UF4-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF1UF4: pred.store.continue:
; VF1UF4-NEXT: [[TMP6:%.*]] = phi i64 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_STORE_IF]] ]
; VF1UF4-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; VF1UF4: pred.store.if7:
+; VF1UF4: pred.store.if4:
; VF1UF4-NEXT: [[INDUCTION1:%.*]] = add i64 [[INDEX]], 1
; VF1UF4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[INDUCTION1]]
; VF1UF4-NEXT: [[TMP8:%.*]] = load i64, i64* [[TMP7]], align 8
; VF1UF4-NEXT: store i64 [[TMP8]], i64* [[B]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; VF1UF4: pred.store.continue8:
+; VF1UF4: pred.store.continue5:
; VF1UF4-NEXT: [[TMP9:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE]] ], [ [[TMP8]], [[PRED_STORE_IF7]] ]
; VF1UF4-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; VF1UF4: pred.store.if9:
+; VF1UF4: pred.store.if6:
; VF1UF4-NEXT: [[INDUCTION2:%.*]] = add i64 [[INDEX]], 2
; VF1UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[INDUCTION2]]
; VF1UF4-NEXT: [[TMP11:%.*]] = load i64, i64* [[TMP10]], align 8
; VF1UF4-NEXT: store i64 [[TMP11]], i64* [[B]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; VF1UF4: pred.store.continue10:
+; VF1UF4: pred.store.continue7:
; VF1UF4-NEXT: [[TMP12:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE8]] ], [ [[TMP11]], [[PRED_STORE_IF9]] ]
; VF1UF4-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; VF1UF4: pred.store.if11:
+; VF1UF4: pred.store.if8:
; VF1UF4-NEXT: [[INDUCTION3:%.*]] = add i64 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[INDUCTION3]]
; VF1UF4-NEXT: [[TMP14:%.*]] = load i64, i64* [[TMP13]], align 8
; VF1UF4-NEXT: store i64 [[TMP14]], i64* [[B]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; VF1UF4: pred.store.continue12:
+; VF1UF4: pred.store.continue9:
; VF1UF4-NEXT: [[TMP15:%.*]] = phi i64 [ poison, [[PRED_STORE_CONTINUE10]] ], [ [[TMP14]], [[PRED_STORE_IF11]] ]
; VF1UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; VF1UF4-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
;
; CHECK-VF1IC2-LABEL: @pred_select_const_i32_from_icmp(
; CHECK-VF1IC2: vector.body:
-; CHECK-VF1IC2: [[VEC_PHI:%.*]] = phi i32 [ 0, %vector.ph ], [ [[PREDPHI:%.*]], %pred.load.continue4 ]
-; CHECK-VF1IC2-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, %vector.ph ], [ [[PREDPHI5:%.*]], %pred.load.continue4 ]
+; CHECK-VF1IC2: [[VEC_PHI:%.*]] = phi i32 [ 0, %vector.ph ], [ [[PREDPHI:%.*]], %pred.load.continue3 ]
+; CHECK-VF1IC2-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, %vector.ph ], [ [[PREDPHI5:%.*]], %pred.load.continue3 ]
; CHECK-VF1IC2: [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[SRC1:%.*]], i64 {{%.*}}
; CHECK-VF1IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[SRC1]], i64 {{%.*}}
; CHECK-VF1IC2-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]], align 4
; CHECK-VF1IC2-NEXT: br label %pred.load.continue
; CHECK-VF1IC2: pred.load.continue:
; CHECK-VF1IC2-NEXT: [[TMP8:%.*]] = phi i32 [ poison, %vector.body ], [ [[TMP7]], %pred.load.if ]
-; CHECK-VF1IC2-NEXT: br i1 [[TMP5]], label %pred.load.if3, label %pred.load.continue4
-; CHECK-VF1IC2: pred.load.if3:
+; CHECK-VF1IC2-NEXT: br i1 [[TMP5]], label %pred.load.if2, label %pred.load.continue3
+; CHECK-VF1IC2: pred.load.if2:
; CHECK-VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[SRC2]], i64 {{%.*}}
; CHECK-VF1IC2-NEXT: [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
-; CHECK-VF1IC2-NEXT: br label %pred.load.continue4
-; CHECK-VF1IC2: pred.load.continue4:
-; CHECK-VF1IC2-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %pred.load.continue ], [ [[TMP10]], %pred.load.if3 ]
+; CHECK-VF1IC2-NEXT: br label %pred.load.continue3
+; CHECK-VF1IC2: pred.load.continue3:
+; CHECK-VF1IC2-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %pred.load.continue ], [ [[TMP10]], %pred.load.if2 ]
; CHECK-VF1IC2-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 2
; CHECK-VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP11]], 2
; CHECK-VF1IC2-NEXT: [[TMP14:%.*]] = select i1 [[TMP12]], i32 1, i32 [[VEC_PHI]]
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK: pred.store.if4:
; CHECK-NEXT: [[INDUCTION1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 [[INDUCTION1]]
; CHECK-NEXT: store i32 0, i32* [[TMP5]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
+; CHECK: pred.store.continue5:
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.if6:
; CHECK-NEXT: [[INDUCTION2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 [[INDUCTION2]]
; CHECK-NEXT: store i32 0, i32* [[TMP6]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
+; CHECK: pred.store.continue7:
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.if8:
; CHECK-NEXT: [[INDUCTION3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 [[INDUCTION3]]
; CHECK-NEXT: store i32 0, i32* [[TMP7]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]