A first step towards modeling preheader and exit blocks in VPlan as well.
Keeping the vector loop in a region allows for changing the VF as we
traverse region boundaries.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D113182
if (VPBB)
VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
else {
- Plan->setEntry(FirstVPBBForBB);
+ auto *TopRegion = new VPRegionBlock("vector loop");
+ TopRegion->setEntry(FirstVPBBForBB);
+ Plan->setEntry(TopRegion);
HeaderVPBB = FirstVPBBForBB;
}
VPBB = FirstVPBBForBB;
}
}
- assert(isa<VPBasicBlock>(Plan->getEntry()) &&
+ assert(isa<VPRegionBlock>(Plan->getEntry()) &&
!Plan->getEntry()->getEntryBasicBlock()->empty() &&
- "entry block must be set to a non-empty VPBasicBlock");
+ "entry block must be set to a VPRegionBlock having a non-empty entry "
+ "VPBasicBlock");
+ cast<VPRegionBlock>(Plan->getEntry())->setExit(VPBB);
RecipeBuilder.fixHeaderPhis();
// ---------------------------------------------------------------------------
; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, i8* %ptr.iv.2, i64 1
;
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN-PHI %ptr.iv.1 = phi %start.1, %ptr.iv.1.next
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<[[PRED2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: loop.1:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]> ir<%and.red.next> ir<%and.red>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: loop.2:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop
; Check for crash exposed by D76992.
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%cond0> = icmp ir<%iv>, ir<13>
; Verify that -vplan-print-in-dot-format option works.
define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
-; CHECK: N0 [label =
+; CHECK: subgraph cluster_N0 {
+; CHECK-NEXT: fontname=Courier
+; CHECK-NEXT: label="\<x1\> vector loop"
+; CHECK-NEXT: N1 [label =
; CHECK-NEXT: "for.body:\l" +
; CHECK-NEXT: " WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, ir\<%iv\>\l" +
define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
; CHECK-LABEL: Checking a loop in "print_call_and_memory"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
%cmp6 = icmp sgt i64 %n, 0
define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalias %x, float* %z) nounwind uwtable {
; CHECK-LABEL: Checking a loop in "print_widen_gep_and_select"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
%cmp6 = icmp sgt i64 %n, 0
define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK-LABEL: Checking a loop in "print_reduction"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %for.body
define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
; CHECK-LABEL: Checking a loop in "print_replicate_predicated_phi"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next
; CHECK-NEXT: WIDEN ir<%cmp> = icmp ir<%i>, ir<5>
; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %for.body
define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-LABEL: Checking a loop in "print_interleave_groups"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, ir<%iv>
; CHECK-NEXT: store ir<%AB.3> to index 3
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %for.body
define void @sink_with_sideeffects(i1 %c, i8* %ptr) {
; CHECK-LABEL: sink_with_sideeffects
; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %tmp0 = phi %tmp6, 0
; CHECK-NEXT: WIDEN-INDUCTION %tmp1 = phi %tmp7, 0
; CHECK: for.inc:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %for.body
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 21, %iv.next
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%c.1> = icmp ir<%iv>, ir<%j>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, ir<%lv.a>
define void @update_multiple_users(i16* noalias %src, i8* noalias %dst, i1 %c) {
; CHECK-LABEL: LV: Checking a loop in "update_multiple_users"
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.header:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: Successor(s): loop.then
define void @sinking_requires_duplication(float* %addr) {
; CHECK-LABEL: LV: Checking a loop in "sinking_requires_duplication"
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.header:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, ir<%iv>