if (L.getHeader()->getParent()->hasOptSize())
return false;
- // Skip cold loops, as unswitching them brings little benefit
- // but increases the code size
- if (PSI && PSI->hasProfileSummary() && BFI &&
- PSI->isFunctionColdInCallGraph(L.getHeader()->getParent(), *BFI)) {
+ // Returns true if Loop L's loop nest is cold, i.e. if the headers of L,
+ // of the loops L is nested in, and of the loops nested in L are all cold.
+ auto IsLoopNestCold = [&](const Loop *L) {
+ // Check L and all of its parent loops.
+ auto *Parent = L;
+ while (Parent) {
+ if (!PSI->isColdBlock(Parent->getHeader(), BFI))
+ return false;
+ Parent = Parent->getParentLoop();
+ }
+ // Next check all loops nested within L.
+ SmallVector<const Loop *, 4> Worklist;
+ Worklist.insert(Worklist.end(), L->getSubLoops().begin(),
+ L->getSubLoops().end());
+ while (!Worklist.empty()) {
+ auto *CurLoop = Worklist.pop_back_val();
+ if (!PSI->isColdBlock(CurLoop->getHeader(), BFI))
+ return false;
+ Worklist.insert(Worklist.end(), CurLoop->getSubLoops().begin(),
+ CurLoop->getSubLoops().end());
+ }
+ return true;
+ };
+
+ // Skip cold loops in cold loop nests, as unswitching them brings little
+ // benefit but increases the code size
+ if (PSI && PSI->hasProfileSummary() && BFI && IsLoopNestCold(&L)) {
LLVM_DEBUG(dbgs() << " Skip cold loop: " << L << "\n");
return false;
}
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -passes='require<profile-summary>,function(loop-mssa(simple-loop-unswitch<nontrivial>))' -S | FileCheck %s
-; This test checks for a crash.
-; RUN: opt < %s -passes=simple-loop-unswitch -aa-pipeline= -disable-output
-declare i32 @a()
-declare i32 @b()
-; Check loops in cold functions will not be applied non-trivial loop unswitch
-define void @f1(i32 %i, i1 %cond, i1 %hot_cond, i1 %cold_cond, ptr %ptr) !prof !0 {
-; CHECK-LABEL: @f1(
+;; Check that non-trivial loop unswitching is not applied to a cold loop in a
+;; cold loop nest.
+
+;; IR was generated from the following loop nest, profiled when called
+;; with M=0 and N=0.
+;; void hotFunction(bool cond, int M, int N, int * A, int *B, int *C) {
+;; for (unsigned j = 0; j < M; j++)
+;; for (unsigned i=0; i < N; i++) {
+;; A[i] = B[i] + C[i];
+;; if (cond) do_something();
+;; }
+;; }
+
+define void @_Z11hotFunctionbiiPiS_S_(i1 %cond, i32 %M, i32 %N, ptr %A, ptr %B, ptr %C) !prof !36 {
+; CHECK-LABEL: define void @_Z11hotFunctionbiiPiS_S_
+; CHECK-SAME: (i1 [[COND:%.*]], i32 [[M:%.*]], i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) !prof [[PROF16:![0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[COLD_LOOP_BEGIN:%.*]]
-; CHECK: cold_loop_begin:
-; CHECK-NEXT: br i1 [[COND:%.*]], label [[COLD_LOOP_A:%.*]], label [[COLD_LOOP_B:%.*]]
-; CHECK: cold_loop_a:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @a()
-; CHECK-NEXT: br label [[COLD_LOOP_LATCH:%.*]]
-; CHECK: cold_loop_b:
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @b()
-; CHECK-NEXT: br label [[COLD_LOOP_LATCH]]
-; CHECK: cold_loop_latch:
-; CHECK-NEXT: [[V2:%.*]] = load i1, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT: br i1 [[V2]], label [[COLD_LOOP_BEGIN]], label [[COLD_LOOP_EXIT:%.*]]
-; CHECK: cold_loop_exit:
+; CHECK-NEXT: [[CMP19_NOT:%.*]] = icmp eq i32 [[M]], 0
+; CHECK-NEXT: br i1 [[CMP19_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_LR_PH:%.*]], !prof [[PROF17:![0-9]+]]
+; CHECK: for.cond1.preheader.lr.ph:
+; CHECK-NEXT: [[CMP217_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK: for.cond1.preheader:
+; CHECK-NEXT: [[J_020:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH]] ], [ [[INC10:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-NEXT: br i1 [[CMP217_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4_PREHEADER:%.*]]
+; CHECK: for.body4.preheader:
+; CHECK-NEXT: br label [[FOR_BODY4:%.*]]
+; CHECK: for.cond.cleanup.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
+; CHECK: for.cond.cleanup:
; CHECK-NEXT: ret void
+; CHECK: for.cond.cleanup3.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3]]
+; CHECK: for.cond.cleanup3:
+; CHECK-NEXT: [[INC10]] = add nuw i32 [[J_020]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT:%.*]] = icmp eq i32 [[INC10]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_COND1_PREHEADER]], !prof [[PROF17]]
+; CHECK: for.body4:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY4_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK: if.then:
+; CHECK-NEXT: tail call void @_Z12do_somethingv()
+; CHECK-NEXT: br label [[FOR_INC]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3_LOOPEXIT:%.*]], label [[FOR_BODY4]]
;
entry:
- br label %cold_loop_begin
+ %cmp19.not = icmp eq i32 %M, 0
+ br i1 %cmp19.not, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph, !prof !37
-cold_loop_begin:
- br i1 %cond, label %cold_loop_a, label %cold_loop_b
+for.cond1.preheader.lr.ph:
+ %cmp217.not = icmp eq i32 %N, 0
+ br label %for.cond1.preheader
-cold_loop_a:
- call i32 @a()
- br label %cold_loop_latch
+for.cond1.preheader:
+ %j.020 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %inc10, %for.cond.cleanup3 ]
+ br i1 %cmp217.not, label %for.cond.cleanup3, label %for.body4
-cold_loop_b:
- call i32 @b()
- br label %cold_loop_latch
+for.cond.cleanup:
+ ret void
-cold_loop_latch:
- %v2 = load i1, ptr %ptr
- br i1 %v2, label %cold_loop_begin, label %cold_loop_exit
+for.cond.cleanup3:
+ %inc10 = add nuw i32 %j.020, 1
+ %exitcond22.not = icmp eq i32 %inc10, %M
+ br i1 %exitcond22.not, label %for.cond.cleanup, label %for.cond1.preheader, !prof !37
-cold_loop_exit:
- ret void
+for.body4:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.cond1.preheader ]
+ %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx6 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx6, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx8 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx8, align 4
+ br i1 %cond, label %if.then, label %for.inc
+
+if.then:
+ tail call void @_Z12do_somethingv()
+ br label %for.inc
+
+for.inc:
+ %wide.trip.count = zext i32 %N to i64
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup3, label %for.body4
}
-!llvm.module.flags = !{!1}
-!0 = !{!"function_entry_count", i64 0}
-!1 = !{i32 1, !"ProfileSummary", !2}
-!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
-!3 = !{!"ProfileFormat", !"InstrProf"}
-!4 = !{!"TotalCount", i64 10000}
-!5 = !{!"MaxCount", i64 10}
-!6 = !{!"MaxInternalCount", i64 1}
-!7 = !{!"MaxFunctionCount", i64 1000}
-!8 = !{!"NumCounts", i64 3}
-!9 = !{!"NumFunctions", i64 3}
-!10 = !{!"DetailedSummary", !11}
-!11 = !{!12, !13, !14}
-!12 = !{i32 10000, i64 100, i32 1}
-!13 = !{i32 999000, i64 100, i32 1}
-!14 = !{i32 999999, i64 1, i32 2}
+declare void @_Z12do_somethingv()
+
+!llvm.module.flags = !{!6}
+
+!6 = !{i32 1, !"ProfileSummary", !7}
+!7 = !{!8, !9, !10, !11, !12, !13, !14, !15, !16, !17}
+!8 = !{!"ProfileFormat", !"InstrProf"}
+!9 = !{!"TotalCount", i64 1002}
+!10 = !{!"MaxCount", i64 1000}
+!11 = !{!"MaxInternalCount", i64 1000}
+!12 = !{!"MaxFunctionCount", i64 1}
+!13 = !{!"NumCounts", i64 6}
+!14 = !{!"NumFunctions", i64 3}
+!15 = !{!"IsPartialProfile", i64 0}
+!16 = !{!"PartialProfileRatio", double 0.000000e+00}
+!17 = !{!"DetailedSummary", !18}
+!18 = !{!19, !31, !34}
+!19 = !{i32 10000, i64 1000, i32 1}
+!31 = !{i32 999000, i64 1000, i32 1}
+!34 = !{i32 999999, i64 1, i32 3}
+!36 = !{!"function_entry_count", i64 1}
+!37 = !{!"branch_weights", i32 1, i32 0}
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -passes='require<profile-summary>,function(loop-mssa(simple-loop-unswitch<nontrivial>))' -S | FileCheck %s
-declare i32 @a()
-declare i32 @b()
+;; Check that non-trivial loop unswitching is applied to a cold loop in a
+;; non-cold loop nest.
-; Check loops will be applied non-trivial loop unswitch in a non-cold function,
-; even loop headers are cold
+;; IR was generated from the following loop nest, profiled when called
+;; with M=1000 and N=0.
+;; void hotFunction(bool cond, int M, int N, int * A, int *B, int *C) {
+;; for (unsigned j = 0; j < M; j++)
+;; for (unsigned i=0; i < N; i++) {
+;; A[i] = B[i] + C[i];
+;; if (cond) do_something();
+;; }
+;; }
-define void @f1(i32 %i, i1 %cond, i1 %hot_cond, i1 %cold_cond, ptr %ptr) !prof !14 {
-; CHECK-LABEL: @f1(
+define void @_Z11hotFunctionbiiPiS_S_(i1 %cond, i32 %M, i32 %N, ptr %A, ptr %B, ptr %C) !prof !36 {
+; CHECK-LABEL: define void @_Z11hotFunctionbiiPiS_S_
+; CHECK-SAME: (i1 [[COND:%.*]], i32 [[M:%.*]], i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) !prof [[PROF33:![0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[ENTRY_COLD_LOOP:%.*]]
-; CHECK: entry_cold_loop:
-; CHECK-NEXT: br i1 [[COLD_COND:%.*]], label [[COLD_LOOP_BEGIN_PREHEADER:%.*]], label [[COLD_LOOP_EXIT:%.*]], !prof [[PROF15:![0-9]+]]
-; CHECK: cold_loop_begin.preheader:
-; CHECK-NEXT: br i1 [[COND:%.*]], label [[COLD_LOOP_BEGIN_PREHEADER_SPLIT_US:%.*]], label [[COLD_LOOP_BEGIN_PREHEADER_SPLIT:%.*]]
-; CHECK: cold_loop_begin.preheader.split.us:
-; CHECK-NEXT: br label [[COLD_LOOP_BEGIN_US:%.*]]
-; CHECK: cold_loop_begin.us:
-; CHECK-NEXT: br label [[COLD_LOOP_A_US:%.*]]
-; CHECK: cold_loop_a.us:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @a()
-; CHECK-NEXT: br label [[COLD_LOOP_LATCH_US:%.*]]
-; CHECK: cold_loop_latch.us:
-; CHECK-NEXT: [[V2_US:%.*]] = load i1, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT: br i1 [[V2_US]], label [[COLD_LOOP_BEGIN_US]], label [[COLD_LOOP_EXIT_LOOPEXIT_SPLIT_US:%.*]]
-; CHECK: cold_loop_exit.loopexit.split.us:
-; CHECK-NEXT: br label [[COLD_LOOP_EXIT_LOOPEXIT:%.*]]
-; CHECK: cold_loop_begin.preheader.split:
-; CHECK-NEXT: br label [[COLD_LOOP_BEGIN:%.*]]
-; CHECK: cold_loop_begin:
-; CHECK-NEXT: br label [[COLD_LOOP_B:%.*]]
-; CHECK: cold_loop_b:
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @b()
-; CHECK-NEXT: br label [[COLD_LOOP_LATCH:%.*]]
-; CHECK: cold_loop_latch:
-; CHECK-NEXT: [[V2:%.*]] = load i1, ptr [[PTR]], align 1
-; CHECK-NEXT: br i1 [[V2]], label [[COLD_LOOP_BEGIN]], label [[COLD_LOOP_EXIT_LOOPEXIT_SPLIT:%.*]]
-; CHECK: cold_loop_exit.loopexit.split:
-; CHECK-NEXT: br label [[COLD_LOOP_EXIT_LOOPEXIT]]
-; CHECK: cold_loop_exit.loopexit:
-; CHECK-NEXT: br label [[COLD_LOOP_EXIT]]
-; CHECK: cold_loop_exit:
+; CHECK-NEXT: [[CMP19_NOT:%.*]] = icmp eq i32 [[M]], 0
+; CHECK-NEXT: br i1 [[CMP19_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_LR_PH:%.*]], !prof [[PROF34:![0-9]+]]
+; CHECK: for.cond1.preheader.lr.ph:
+; CHECK-NEXT: [[CMP217_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP217_NOT]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT:%.*]], !prof [[PROF35:![0-9]+]]
+; CHECK: for.cond1.preheader.lr.ph.split.us:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK: for.cond1.preheader.us:
+; CHECK-NEXT: [[J_020_US:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US]] ], [ [[INC10_US:%.*]], [[FOR_COND_CLEANUP3_US:%.*]] ]
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_US]]
+; CHECK: for.cond.cleanup3.us:
+; CHECK-NEXT: [[INC10_US]] = add nuw i32 [[J_020_US]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT_US:%.*]] = icmp eq i32 [[INC10_US]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US]], !prof [[PROF34]]
+; CHECK: for.cond.cleanup.loopexit.split.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split:
+; CHECK-NEXT: br i1 [[COND]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split.split.us:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER_US1:%.*]]
+; CHECK: for.cond1.preheader.us1:
+; CHECK-NEXT: [[J_020_US2:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT_US]] ], [ [[INC10_US4:%.*]], [[FOR_COND_CLEANUP3_US3:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_US:%.*]]
+; CHECK: for.cond.cleanup3.us3:
+; CHECK-NEXT: [[INC10_US4]] = add nuw i32 [[J_020_US2]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT_US5:%.*]] = icmp eq i32 [[INC10_US4]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT_US5]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US1]], !prof [[PROF34]]
+; CHECK: for.body4.preheader.us:
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_SPLIT_US_US:%.*]]
+; CHECK: for.cond.cleanup3.loopexit.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_US3]]
+; CHECK: for.body4.preheader.split.us.us:
+; CHECK-NEXT: br label [[FOR_BODY4_US_US:%.*]]
+; CHECK: for.body4.us.us:
+; CHECK-NEXT: [[INDVARS_IV_US_US:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_US_US:%.*]], [[FOR_INC_US_US:%.*]] ], [ 0, [[FOR_BODY4_PREHEADER_SPLIT_US_US]] ]
+; CHECK-NEXT: [[ARRAYIDX_US_US:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX_US_US]], align 4
+; CHECK-NEXT: [[ARRAYIDX6_US_US:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX6_US_US]], align 4
+; CHECK-NEXT: [[ADD_US_US:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[ARRAYIDX8_US_US:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: store i32 [[ADD_US_US]], ptr [[ARRAYIDX8_US_US]], align 4
+; CHECK-NEXT: br label [[IF_THEN_US_US:%.*]]
+; CHECK: if.then.us.us:
+; CHECK-NEXT: tail call void @_Z12do_somethingv()
+; CHECK-NEXT: br label [[FOR_INC_US_US]]
+; CHECK: for.inc.us.us:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT_US_US:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[INDVARS_IV_NEXT_US_US]] = add nuw nsw i64 [[INDVARS_IV_US_US]], 1
+; CHECK-NEXT: [[EXITCOND_NOT_US_US:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_US_US]], [[WIDE_TRIP_COUNT_US_US]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT_US_US]], label [[FOR_COND_CLEANUP3_LOOPEXIT_SPLIT_US_US:%.*]], label [[FOR_BODY4_US_US]], !prof [[PROF35]]
+; CHECK: for.cond.cleanup3.loopexit.split.us.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_LOOPEXIT_US:%.*]]
+; CHECK: for.cond.cleanup.loopexit.split.split.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split.split:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK: for.cond1.preheader:
+; CHECK-NEXT: [[J_020:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT]] ], [ [[INC10:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER:%.*]]
+; CHECK: for.body4.preheader:
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_SPLIT:%.*]]
+; CHECK: for.body4.preheader.split:
+; CHECK-NEXT: br label [[FOR_BODY4:%.*]]
+; CHECK: for.cond.cleanup.loopexit.split.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT]]
+; CHECK: for.cond.cleanup.loopexit.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: for.cond.cleanup.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
+; CHECK: for.cond.cleanup:
; CHECK-NEXT: ret void
+; CHECK: for.cond.cleanup3.loopexit.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_LOOPEXIT:%.*]]
+; CHECK: for.cond.cleanup3.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3]]
+; CHECK: for.cond.cleanup3:
+; CHECK-NEXT: [[INC10]] = add nuw i32 [[J_020]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT:%.*]] = icmp eq i32 [[INC10]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_SPLIT:%.*]], label [[FOR_COND1_PREHEADER]], !prof [[PROF34]]
+; CHECK: for.body4:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY4_PREHEADER_SPLIT]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT: br label [[FOR_INC]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3_LOOPEXIT_SPLIT:%.*]], label [[FOR_BODY4]], !prof [[PROF35]]
;
entry:
- br label %entry_cold_loop
+ %cmp19.not = icmp eq i32 %M, 0
+ br i1 %cmp19.not, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph, !prof !37
-entry_cold_loop:
- br i1 %cold_cond, label %cold_loop_begin, label %cold_loop_exit, !prof !15
+for.cond1.preheader.lr.ph:
+ %cmp217.not = icmp eq i32 %N, 0
+ br label %for.cond1.preheader
-cold_loop_begin:
- br i1 %cond, label %cold_loop_a, label %cold_loop_b
+for.cond1.preheader:
+ %j.020 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %inc10, %for.cond.cleanup3 ]
+ br i1 %cmp217.not, label %for.cond.cleanup3, label %for.body4, !prof !38
-cold_loop_a:
- %0 = call i32 @a()
- br label %cold_loop_latch
+for.cond.cleanup:
+ ret void
-cold_loop_b:
- %1 = call i32 @b()
- br label %cold_loop_latch
+for.cond.cleanup3:
+ %inc10 = add nuw i32 %j.020, 1
+ %exitcond22.not = icmp eq i32 %inc10, %M
+ br i1 %exitcond22.not, label %for.cond.cleanup, label %for.cond1.preheader, !prof !37
-cold_loop_latch:
- %v2 = load i1, ptr %ptr
- br i1 %v2, label %cold_loop_begin, label %cold_loop_exit
+for.body4:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.cond1.preheader ]
+ %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx6 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx6, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx8 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx8, align 4
+ br i1 %cond, label %if.then, label %for.inc
-cold_loop_exit:
- ret void
+if.then:
+ tail call void @_Z12do_somethingv()
+ br label %for.inc
+
+for.inc:
+ %wide.trip.count = zext i32 %N to i64
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup3, label %for.body4, !prof !38
}
-!llvm.module.flags = !{!0}
+declare void @_Z12do_somethingv()
+
+!llvm.module.flags = !{!6}
-!0 = !{i32 1, !"ProfileSummary", !1}
-!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
-!2 = !{!"ProfileFormat", !"InstrProf"}
-!3 = !{!"TotalCount", i64 10000}
-!4 = !{!"MaxCount", i64 10}
-!5 = !{!"MaxInternalCount", i64 1}
-!6 = !{!"MaxFunctionCount", i64 1000}
-!7 = !{!"NumCounts", i64 3}
-!8 = !{!"NumFunctions", i64 3}
-!9 = !{!"DetailedSummary", !10}
-!10 = !{!11, !12, !13}
-!11 = !{i32 10000, i64 100, i32 1}
-!12 = !{i32 999000, i64 100, i32 1}
-!13 = !{i32 999999, i64 1, i32 2}
-!14 = !{!"function_entry_count", i64 400}
-!15 = !{!"branch_weights", i32 0, i32 100}
+!6 = !{i32 1, !"ProfileSummary", !7}
+!7 = !{!8, !9, !10, !11, !12, !13, !14, !15, !16, !17}
+!8 = !{!"ProfileFormat", !"InstrProf"}
+!9 = !{!"TotalCount", i64 1002}
+!10 = !{!"MaxCount", i64 1000}
+!11 = !{!"MaxInternalCount", i64 1000}
+!12 = !{!"MaxFunctionCount", i64 1}
+!13 = !{!"NumCounts", i64 6}
+!14 = !{!"NumFunctions", i64 3}
+!15 = !{!"IsPartialProfile", i64 0}
+!16 = !{!"PartialProfileRatio", double 0.000000e+00}
+!17 = !{!"DetailedSummary", !18}
+!18 = !{!19, !31, !34}
+!19 = !{i32 10000, i64 1000, i32 1}
+!31 = !{i32 999000, i64 1000, i32 1}
+!34 = !{i32 999999, i64 1, i32 3}
+!36 = !{!"function_entry_count", i64 1}
+!37 = !{!"branch_weights", i32 1, i32 1000}
+!38 = !{!"branch_weights", i32 1000, i32 0}
--- /dev/null
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+
+; RUN: opt < %s -passes='require<profile-summary>,function(loop-mssa(simple-loop-unswitch<nontrivial>))' -S | FileCheck %s
+
+;; Check that non-trivial loop unswitching is applied to a non-cold loop in a
+;; non-cold loop nest.
+
+;; IR was generated from the following loop nest, profiled when called
+;; with M=1000 and N=10.
+;; void hotFunction(bool cond, int M, int N, int * A, int *B, int *C) {
+;; for (unsigned j = 0; j < M; j++)
+;; for (unsigned i=0; i < N; i++) {
+;; A[i] = B[i] + C[i];
+;; if (cond) do_something();
+;; }
+;; }
+
+define void @_Z11hotFunctionbiiPiS_S_(i1 %cond, i32 %M, i32 %N, ptr %A, ptr %B, ptr %C) !prof !36 {
+; CHECK-LABEL: define void @_Z11hotFunctionbiiPiS_S_
+; CHECK-SAME: (i1 [[COND:%.*]], i32 [[M:%.*]], i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) !prof [[PROF18:![0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP19_NOT:%.*]] = icmp eq i32 [[M]], 0
+; CHECK-NEXT: br i1 [[CMP19_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_LR_PH:%.*]], !prof [[PROF19:![0-9]+]]
+; CHECK: for.cond1.preheader.lr.ph:
+; CHECK-NEXT: [[CMP217_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP217_NOT]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT:%.*]], !prof [[PROF20:![0-9]+]]
+; CHECK: for.cond1.preheader.lr.ph.split.us:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK: for.cond1.preheader.us:
+; CHECK-NEXT: [[J_020_US:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_US]] ], [ [[INC10_US:%.*]], [[FOR_COND_CLEANUP3_US:%.*]] ]
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_US]]
+; CHECK: for.cond.cleanup3.us:
+; CHECK-NEXT: [[INC10_US]] = add nuw i32 [[J_020_US]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT_US:%.*]] = icmp eq i32 [[INC10_US]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US]], !prof [[PROF19]]
+; CHECK: for.cond.cleanup.loopexit.split.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split:
+; CHECK-NEXT: br i1 [[COND]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split.split.us:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER_US1:%.*]]
+; CHECK: for.cond1.preheader.us1:
+; CHECK-NEXT: [[J_020_US2:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT_US]] ], [ [[INC10_US4:%.*]], [[FOR_COND_CLEANUP3_US3:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_US:%.*]]
+; CHECK: for.cond.cleanup3.us3:
+; CHECK-NEXT: [[INC10_US4]] = add nuw i32 [[J_020_US2]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT_US5:%.*]] = icmp eq i32 [[INC10_US4]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT_US5]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US1]], !prof [[PROF19]]
+; CHECK: for.body4.preheader.us:
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_SPLIT_US_US:%.*]]
+; CHECK: for.cond.cleanup3.loopexit.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_US3]]
+; CHECK: for.body4.preheader.split.us.us:
+; CHECK-NEXT: br label [[FOR_BODY4_US_US:%.*]]
+; CHECK: for.body4.us.us:
+; CHECK-NEXT: [[INDVARS_IV_US_US:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_US_US:%.*]], [[FOR_INC_US_US:%.*]] ], [ 0, [[FOR_BODY4_PREHEADER_SPLIT_US_US]] ]
+; CHECK-NEXT: [[ARRAYIDX_US_US:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX_US_US]], align 4
+; CHECK-NEXT: [[ARRAYIDX6_US_US:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX6_US_US]], align 4
+; CHECK-NEXT: [[ADD_US_US:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[ARRAYIDX8_US_US:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_US_US]]
+; CHECK-NEXT: store i32 [[ADD_US_US]], ptr [[ARRAYIDX8_US_US]], align 4
+; CHECK-NEXT: br label [[IF_THEN_US_US:%.*]]
+; CHECK: if.then.us.us:
+; CHECK-NEXT: tail call void @_Z12do_somethingv()
+; CHECK-NEXT: br label [[FOR_INC_US_US]]
+; CHECK: for.inc.us.us:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT_US_US:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[INDVARS_IV_NEXT_US_US]] = add nuw nsw i64 [[INDVARS_IV_US_US]], 1
+; CHECK-NEXT: [[EXITCOND_NOT_US_US:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_US_US]], [[WIDE_TRIP_COUNT_US_US]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT_US_US]], label [[FOR_COND_CLEANUP3_LOOPEXIT_SPLIT_US_US:%.*]], label [[FOR_BODY4_US_US]], !prof [[PROF20]]
+; CHECK: for.cond.cleanup3.loopexit.split.us.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_LOOPEXIT_US:%.*]]
+; CHECK: for.cond.cleanup.loopexit.split.split.us:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT:%.*]]
+; CHECK: for.cond1.preheader.lr.ph.split.split:
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK: for.cond1.preheader:
+; CHECK-NEXT: [[J_020:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_LR_PH_SPLIT_SPLIT]] ], [ [[INC10:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER:%.*]]
+; CHECK: for.body4.preheader:
+; CHECK-NEXT: br label [[FOR_BODY4_PREHEADER_SPLIT:%.*]]
+; CHECK: for.body4.preheader.split:
+; CHECK-NEXT: br label [[FOR_BODY4:%.*]]
+; CHECK: for.cond.cleanup.loopexit.split.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT]]
+; CHECK: for.cond.cleanup.loopexit.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: for.cond.cleanup.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+; CHECK: for.cond.cleanup3.loopexit.split:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3_LOOPEXIT:%.*]]
+; CHECK: for.cond.cleanup3.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP3]]
+; CHECK: for.cond.cleanup3:
+; CHECK-NEXT: [[INC10]] = add nuw i32 [[J_020]], 1
+; CHECK-NEXT: [[EXITCOND22_NOT:%.*]] = icmp eq i32 [[INC10]], [[M]]
+; CHECK-NEXT: br i1 [[EXITCOND22_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_SPLIT:%.*]], label [[FOR_COND1_PREHEADER]], !prof [[PROF19]]
+; CHECK: for.body4:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY4_PREHEADER_SPLIT]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX8]], align 4
+; CHECK-NEXT: br label [[FOR_INC]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3_LOOPEXIT_SPLIT:%.*]], label [[FOR_BODY4]], !prof [[PROF20]]
+;
+entry:
+ %cmp19.not = icmp eq i32 %M, 0
+ br i1 %cmp19.not, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph, !prof !37
+
+for.cond1.preheader.lr.ph:
+ %cmp217.not = icmp eq i32 %N, 0
+ br label %for.cond1.preheader
+
+for.cond1.preheader:
+ %j.020 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %inc10, %for.cond.cleanup3 ]
+ br i1 %cmp217.not, label %for.cond.cleanup3, label %for.body4, !prof !38
+
+for.cond.cleanup:
+ ret void
+
+for.cond.cleanup3:
+ %inc10 = add nuw i32 %j.020, 1
+ %exitcond22.not = icmp eq i32 %inc10, %M
+ br i1 %exitcond22.not, label %for.cond.cleanup, label %for.cond1.preheader, !prof !37
+
+for.body4:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.cond1.preheader ]
+ %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx6 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx6, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx8 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx8, align 4
+ br i1 %cond, label %if.then, label %for.inc
+
+if.then:
+ tail call void @_Z12do_somethingv()
+ br label %for.inc
+
+for.inc:
+ %wide.trip.count = zext i32 %N to i64
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup3, label %for.body4, !prof !38
+}
+
+declare void @_Z12do_somethingv()
+
+!llvm.module.flags = !{!6}
+
+!6 = !{i32 1, !"ProfileSummary", !7}
+!7 = !{!8, !9, !10, !11, !12, !13, !14, !15, !16, !17}
+!8 = !{!"ProfileFormat", !"InstrProf"}
+!9 = !{!"TotalCount", i64 1002}
+!10 = !{!"MaxCount", i64 1000}
+!11 = !{!"MaxInternalCount", i64 1000}
+!12 = !{!"MaxFunctionCount", i64 1}
+!13 = !{!"NumCounts", i64 6}
+!14 = !{!"NumFunctions", i64 3}
+!15 = !{!"IsPartialProfile", i64 0}
+!16 = !{!"PartialProfileRatio", double 0.000000e+00}
+!17 = !{!"DetailedSummary", !18}
+!18 = !{!19, !29, !30, !32, !34}
+!19 = !{i32 10000, i64 10000, i32 3}
+!29 = !{i32 950000, i64 10000, i32 3}
+!30 = !{i32 990000, i64 1000, i32 4}
+!32 = !{i32 999900, i64 1000, i32 4}
+!34 = !{i32 999999, i64 1, i32 6}
+!36 = !{!"function_entry_count", i64 1}
+!37 = !{!"branch_weights", i32 1, i32 1000}
+!38 = !{!"branch_weights", i32 1000, i32 10000}