From 458c230b5ef893238d2471fcff27cd275e8026d5 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Wed, 30 Jun 2021 15:40:45 -0700
Subject: [PATCH] GlobalISel/AArch64: don't optimize away redundant branches at
 -O0

This patch prevents GlobalISel from optimizing out redundant branch
instructions when compiling without optimizations.

The motivating example is code like the following common pattern in
Swift, where users expect to be able to set a breakpoint on the early
exit:

public func f(b: Bool) {
  guard b else {
    return // I would like to set a breakpoint here.
  }
  ...
}

The patch modifies two places in GlobalISEL: The first one is in
IRTranslator.cpp where the removal of redundant branches is made
conditional on the optimization level. The second one is in
AArch64InstructionSelector.cpp where an -O0 *only* optimization is
being removed.

Disabling these optimizations increases code size at -O0 by
~8%. However, doing so improves debuggability, and debug builds are
the primary reason why developers compile without optimizations. We
thus concluded that this is the right trade-off.

rdar://79515454

Differential Revision: https://reviews.llvm.org/D105238
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp       |  2 +-
 .../AArch64/GISel/AArch64InstructionSelector.cpp   | 13 +-----
 .../CodeGen/AArch64/GlobalISel/arm64-atomic.ll     | 24 ++++++++---
 .../AArch64/GlobalISel/arm64-irtranslator.ll       |  3 --
 llvm/test/CodeGen/AArch64/unwind-preserved.ll      |  3 +-
 .../Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll | 10 +++--
 .../GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll | 16 ++++++++
 .../GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll | 16 ++++++++
 llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll   | 18 ++++++++
 llvm/test/DebugInfo/AArch64/fallthrough-branch.ll  | 48 ++++++++++++++++++++++
 10 files changed, 127 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/DebugInfo/AArch64/fallthrough-branch.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 73b7637..d4880f0 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -566,7 +566,7 @@ bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
 
   if (BrInst.isUnconditional()) {
     // If the unconditional target is the layout successor, fallthrough.
-    if (!CurMBB.isLayoutSuccessor(Succ0MBB))
+    if (OptLevel == CodeGenOpt::None || !CurMBB.isLayoutSuccessor(Succ0MBB))
       MIRBuilder.buildBr(*Succ0MBB);
 
     // Link successors.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index c0c00b3..1ff75f4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2093,17 +2093,8 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
     I.eraseFromParent();
     return true;
   }
-  case TargetOpcode::G_BR: {
-    // If the branch jumps to the fallthrough block, don't bother emitting it.
-    // Only do this for -O0 for a good code size improvement, because when
-    // optimizations are enabled we want to leave this choice to
-    // MachineBlockPlacement.
-    bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOpt::None;
-    if (EnableOpt || !MBB.isLayoutSuccessor(I.getOperand(0).getMBB()))
-      return false;
-    I.eraseFromParent();
-    return true;
-  }
+  case TargetOpcode::G_BR:
+    return false;
   case TargetOpcode::G_SHL:
     return earlySelectSHL(I, MRI);
   case TargetOpcode::G_CONSTANT: {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
index d32d5f0..86e03ce 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll
@@ -329,6 +329,7 @@ define i32 @fetch_and_nand(i32* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    ldr w8, [x0]
 ; CHECK-NOLSE-O0-NEXT:    str w8, [sp, #28] ; 4-byte Folded Spill
+; CHECK-NOLSE-O0-NEXT:  b LBB6_1
 ; CHECK-NOLSE-O0-NEXT:  LBB6_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O0-NEXT:    ; =>This Loop Header: Depth=1
 ; CHECK-NOLSE-O0-NEXT:    ; Child Loop BB6_2 Depth 2
@@ -353,7 +354,8 @@ define i32 @fetch_and_nand(i32* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    cset w8, eq
 ; CHECK-NOLSE-O0-NEXT:    str w9, [sp, #28] ; 4-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    tbz w8, #0, LBB6_1
-; CHECK-NOLSE-O0-NEXT:  ; %bb.5: ; %atomicrmw.end
+; CHECK-NOLSE-O0-NEXT:    b LBB6_5
+; CHECK-NOLSE-O0-NEXT:  LBB6_5: ; %atomicrmw.end
 ; CHECK-NOLSE-O0-NEXT:    ldr w0, [sp, #12] ; 4-byte Folded Reload
 ; CHECK-NOLSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-NOLSE-O0-NEXT:    ret
@@ -377,6 +379,7 @@ define i32 @fetch_and_nand(i32* %p) #0 {
 ; CHECK-LSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-LSE-O0-NEXT:    ldr w8, [x0]
 ; CHECK-LSE-O0-NEXT:    str w8, [sp, #28] ; 4-byte Folded Spill
+; CHECK-LSE-O0-NEXT:    b LBB6_1
 ; CHECK-LSE-O0-NEXT:  LBB6_1: ; %atomicrmw.start
 ; CHECK-LSE-O0-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-LSE-O0-NEXT:    ldr w8, [sp, #28] ; 4-byte Folded Reload
@@ -390,7 +393,8 @@ define i32 @fetch_and_nand(i32* %p) #0 {
 ; CHECK-LSE-O0-NEXT:    cset w8, eq
 ; CHECK-LSE-O0-NEXT:    str w9, [sp, #28] ; 4-byte Folded Spill
 ; CHECK-LSE-O0-NEXT:    tbz w8, #0, LBB6_1
-; CHECK-LSE-O0-NEXT:  ; %bb.2: ; %atomicrmw.end
+; CHECK-LSE-O0-NEXT:    b LBB6_2
+; CHECK-LSE-O0-NEXT:  LBB6_2: ; %atomicrmw.end
 ; CHECK-LSE-O0-NEXT:    ldr w0, [sp, #12] ; 4-byte Folded Reload
 ; CHECK-LSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-LSE-O0-NEXT:    ret
@@ -418,6 +422,7 @@ define i64 @fetch_and_nand_64(i64* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    ldr x8, [x0]
 ; CHECK-NOLSE-O0-NEXT:    str x8, [sp, #24] ; 8-byte Folded Spill
+; CHECK-NOLSE-O0-NEXT:    b LBB7_1
 ; CHECK-NOLSE-O0-NEXT:  LBB7_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O0-NEXT:    ; =>This Loop Header: Depth=1
 ; CHECK-NOLSE-O0-NEXT:    ; Child Loop BB7_2 Depth 2
@@ -442,7 +447,8 @@ define i64 @fetch_and_nand_64(i64* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    cset w8, eq
 ; CHECK-NOLSE-O0-NEXT:    str x9, [sp, #24] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    tbz w8, #0, LBB7_1
-; CHECK-NOLSE-O0-NEXT:  ; %bb.5: ; %atomicrmw.end
+; CHECK-NOLSE-O0-NEXT:    b LBB7_5
+; CHECK-NOLSE-O0-NEXT:  LBB7_5: ; %atomicrmw.end
 ; CHECK-NOLSE-O0-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-NOLSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-NOLSE-O0-NEXT:    ret
@@ -466,6 +472,7 @@ define i64 @fetch_and_nand_64(i64* %p) #0 {
 ; CHECK-LSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-LSE-O0-NEXT:    ldr x8, [x0]
 ; CHECK-LSE-O0-NEXT:    str x8, [sp, #24] ; 8-byte Folded Spill
+; CHECK-LSE-O0-NEXT:    b LBB7_1
 ; CHECK-LSE-O0-NEXT:  LBB7_1: ; %atomicrmw.start
 ; CHECK-LSE-O0-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-LSE-O0-NEXT:    ldr x8, [sp, #24] ; 8-byte Folded Reload
@@ -479,7 +486,8 @@ define i64 @fetch_and_nand_64(i64* %p) #0 {
 ; CHECK-LSE-O0-NEXT:    cset w8, eq
 ; CHECK-LSE-O0-NEXT:    str x9, [sp, #24] ; 8-byte Folded Spill
 ; CHECK-LSE-O0-NEXT:    tbz w8, #0, LBB7_1
-; CHECK-LSE-O0-NEXT:  ; %bb.2: ; %atomicrmw.end
+; CHECK-LSE-O0-NEXT:    b LBB7_2
+; CHECK-LSE-O0-NEXT:  LBB7_2: ; %atomicrmw.end
 ; CHECK-LSE-O0-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-LSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-LSE-O0-NEXT:    ret
@@ -507,6 +515,7 @@ define i32 @fetch_and_or(i32* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    ldr w8, [x0]
 ; CHECK-NOLSE-O0-NEXT:    str w8, [sp, #28] ; 4-byte Folded Spill
+; CHECK-NOLSE-O0-NEXT:    b LBB8_1
 ; CHECK-NOLSE-O0-NEXT:  LBB8_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O0-NEXT:    ; =>This Loop Header: Depth=1
 ; CHECK-NOLSE-O0-NEXT:    ; Child Loop BB8_2 Depth 2
@@ -531,7 +540,8 @@ define i32 @fetch_and_or(i32* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    cset w8, eq
 ; CHECK-NOLSE-O0-NEXT:    str w9, [sp, #28] ; 4-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    tbz w8, #0, LBB8_1
-; CHECK-NOLSE-O0-NEXT:  ; %bb.5: ; %atomicrmw.end
+; CHECK-NOLSE-O0-NEXT:    LBB8_5
+; CHECK-NOLSE-O0-NEXT:  LBB8_5: ; %atomicrmw.end
 ; CHECK-NOLSE-O0-NEXT:    ldr w0, [sp, #12] ; 4-byte Folded Reload
 ; CHECK-NOLSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-NOLSE-O0-NEXT:    ret
@@ -570,6 +580,7 @@ define i64 @fetch_and_or_64(i64* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    str x0, [sp, #16] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    ldr x8, [x0]
 ; CHECK-NOLSE-O0-NEXT:    str x8, [sp, #24] ; 8-byte Folded Spill
+; CHECK-NOLSE-O0-NEXT:    b LBB9_1
 ; CHECK-NOLSE-O0-NEXT:  LBB9_1: ; %atomicrmw.start
 ; CHECK-NOLSE-O0-NEXT:    ; =>This Loop Header: Depth=1
 ; CHECK-NOLSE-O0-NEXT:    ; Child Loop BB9_2 Depth 2
@@ -593,7 +604,8 @@ define i64 @fetch_and_or_64(i64* %p) #0 {
 ; CHECK-NOLSE-O0-NEXT:    cset w8, eq
 ; CHECK-NOLSE-O0-NEXT:    str x9, [sp, #24] ; 8-byte Folded Spill
 ; CHECK-NOLSE-O0-NEXT:    tbz w8, #0, LBB9_1
-; CHECK-NOLSE-O0-NEXT:  ; %bb.5: ; %atomicrmw.end
+; CHECK-NOLSE-O0-NEXT:    b LBB9_5
+; CHECK-NOLSE-O0-NEXT:  LBB9_5: ; %atomicrmw.end
 ; CHECK-NOLSE-O0-NEXT:    ldr x0, [sp, #8] ; 8-byte Folded Reload
 ; CHECK-NOLSE-O0-NEXT:    add sp, sp, #32 ; =32
 ; CHECK-NOLSE-O0-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index b05aa3f..d8a94c4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -87,8 +87,6 @@ bb2:
 ; CHECK: body:
 ; CHECK: bb.{{[0-9]+}}.{{[a-zA-Z0-9.]+}}:
 ; CHECK-NEXT: successors: %[[END:bb.[0-9]+]](0x80000000)
-; We don't emit a branch here, as we can fallthrough to the successor.
-; CHECK-NOT: G_BR
 ; CHECK: [[END]].{{[a-zA-Z0-9.]+}}:
 ; CHECK-NEXT: RET_ReallyLR
 define void @uncondbr_fallthrough() {
@@ -137,7 +135,6 @@ false:
 ; CHECK: bb.{{[0-9]+.[a-zA-Z0-9.]+}}:
 ; Make sure we have one successor
 ; CHECK-NEXT: successors: %[[BB_L1:bb.[0-9]+]](0x80000000)
-; CHECK-NOT: G_BR
 ;
 ; Check basic block L1 has 2 successors: BBL1 and BBL2
 ; CHECK: [[BB_L1]].{{[a-zA-Z0-9.]+}} (address-taken):
diff --git a/llvm/test/CodeGen/AArch64/unwind-preserved.ll b/llvm/test/CodeGen/AArch64/unwind-preserved.ll
index 33bbdfa..0652b7b 100644
--- a/llvm/test/CodeGen/AArch64/unwind-preserved.ll
+++ b/llvm/test/CodeGen/AArch64/unwind-preserved.ll
@@ -365,7 +365,8 @@ define aarch64_vector_pcs <4 x i32> @invoke_callee_may_throw_neon(<4 x i32> %v)
 ; GISEL-NEXT:    bl may_throw_neon
 ; GISEL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; GISEL-NEXT:  .Ltmp4:
-; GISEL-NEXT:  // %bb.1: // %.Lcontinue
+; GISEL-NEXT:    b .LBB1_1 
+; GISEL-NEXT:  .LBB1_1: // %.Lcontinue
 ; GISEL-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload
 ; GISEL-NEXT:    ldp q9, q8, [sp, #256] // 32-byte Folded Reload
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
index 804a148..30a1b51 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/jump_table_and_brjt.ll
@@ -55,8 +55,9 @@ define i32 @mod4_0_to_11(i32 %a) {
 ; MIPS32-NEXT:    jr $ra
 ; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_6: # %sw.default
-; MIPS32-NEXT:    .insn
-; MIPS32-NEXT:  # %bb.7: # %sw.epilog
+; MIPS32-NEXT:    j $BB0_7
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  $BB0_7: # %sw.epilog
 ; MIPS32-NEXT:    lw $1, 8($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ori $3, $zero, 8
@@ -158,8 +159,9 @@ define i32 @mod4_0_to_11(i32 %a) {
 ; MIPS32_PIC-NEXT:    jr $ra
 ; MIPS32_PIC-NEXT:    nop
 ; MIPS32_PIC-NEXT:  $BB0_6: # %sw.default
-; MIPS32_PIC-NEXT:    .insn
-; MIPS32_PIC-NEXT:  # %bb.7: # %sw.epilog
+; MIPS32_PIC-NEXT:    b $BB0_7
+; MIPS32_PIC-NEXT:    nop
+; MIPS32_PIC-NEXT:  $BB0_7: # %sw.epilog
 ; MIPS32_PIC-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
 ; MIPS32_PIC-NEXT:    lw $2, 12($sp) # 4-byte Folded Reload
 ; MIPS32_PIC-NEXT:    ori $3, $zero, 8
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
index ce46bed..0ff034b 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s32.ll
@@ -57,6 +57,8 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB0_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
@@ -93,6 +95,8 @@ define void @long_chain_ambiguous_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32*
 ; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB0_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 24($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
@@ -244,6 +248,8 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB1_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $2, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 32($sp) # 4-byte Folded Reload
@@ -282,6 +288,8 @@ define void @long_chain_i32_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i32* %a, i32* %
 ; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB1_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
@@ -432,6 +440,8 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB2_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
@@ -468,6 +478,8 @@ define void @long_chain_ambiguous_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, flo
 ; MIPS32-NEXT:    lw $1, 36($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($1)
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB2_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 24($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
@@ -620,6 +632,8 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    lw $1, 44($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lwc1 $f0, 0($1)
 ; MIPS32-NEXT:    swc1 $f0, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB3_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_9: # %b.PHI.1
 ; MIPS32-NEXT:    lwc1 $f0, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 32($sp) # 4-byte Folded Reload
@@ -658,6 +672,8 @@ define void @long_chain_float_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, float* %a, fl
 ; MIPS32-NEXT:    lw $1, 40($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lwc1 $f0, 0($1)
 ; MIPS32-NEXT:    swc1 $f0, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB3_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 28($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lwc1 $f0, 4($sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
index 2a5afd5..7650194 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/long_ambiguous_chain_s64.ll
@@ -57,6 +57,8 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    lw $1, 64($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 32($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB0_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 32($sp) # 8-byte Folded Reload
@@ -93,6 +95,8 @@ define void @long_chain_ambiguous_i64_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64*
 ; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB0_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 48($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
@@ -250,6 +254,8 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    lw $1, 4($1)
 ; MIPS32-NEXT:    sw $2, 40($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $1, 44($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB1_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $2, 76($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
@@ -299,6 +305,8 @@ define void @long_chain_i64_in_gpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, i64* %a, i64* %
 ; MIPS32-NEXT:    lw $1, 4($1)
 ; MIPS32-NEXT:    sw $2, 8($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB1_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $3, 8($sp) # 4-byte Folded Reload
@@ -463,6 +471,8 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    lw $1, 64($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 32($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB2_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_9: # %b.PHI.1
 ; MIPS32-NEXT:    lw $1, 52($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 32($sp) # 8-byte Folded Reload
@@ -499,6 +509,8 @@ define void @long_chain_ambiguous_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, do
 ; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB2_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 48($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
@@ -653,6 +665,8 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    lw $1, 72($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 40($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB3_9
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_9: # %b.PHI.1
 ; MIPS32-NEXT:    ldc1 $f0, 80($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 60($sp) # 4-byte Folded Reload
@@ -691,6 +705,8 @@ define void @long_chain_double_in_fpr(i1 %cnd0, i1 %cnd1, i1 %cnd2, double* %a,
 ; MIPS32-NEXT:    lw $1, 68($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($1)
 ; MIPS32-NEXT:    sdc1 $f0, 8($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB3_16
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_16: # %b.PHI.2
 ; MIPS32-NEXT:    lw $1, 56($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 8($sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
index d44023b..100ea22 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/phi.ll
@@ -22,6 +22,8 @@ define i1 @phi_i1(i1 %cnd, i1 %a, i1 %b) {
 ; MIPS32-NEXT:  $BB0_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB0_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB0_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 16
@@ -62,6 +64,8 @@ define i8 @phi_i8(i1 %cnd, i8 %a, i8 %b) {
 ; MIPS32-NEXT:  $BB1_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB1_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB1_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 16
@@ -102,6 +106,8 @@ define i16 @phi_i16(i1 %cnd, i16 %a, i16 %b) {
 ; MIPS32-NEXT:  $BB2_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB2_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB2_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 16
@@ -142,6 +148,8 @@ define i32 @phi_i32(i1 %cnd, i32 %a, i32 %b) {
 ; MIPS32-NEXT:  $BB3_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB3_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB3_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 16
@@ -192,6 +200,8 @@ define i64 @phi_i64(i1 %cnd, i64 %a, i64 %b) {
 ; MIPS32-NEXT:    lw $2, 16($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $2, 0($sp) # 4-byte Folded Spill
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB4_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB4_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 0($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $3, 4($sp) # 4-byte Folded Reload
@@ -236,6 +246,8 @@ define void @phi_ambiguous_i64_in_fpr(i1 %cnd, i64* %i64_ptr_a, i64* %i64_ptr_b,
 ; MIPS32-NEXT:  $BB5_3: # %cond.false
 ; MIPS32-NEXT:    ldc1 $f0, 24($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    sdc1 $f0, 0($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB5_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB5_4: # %cond.end
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    ldc1 $f0, 0($sp) # 8-byte Folded Reload
@@ -281,6 +293,8 @@ define float @phi_float(i1 %cnd, float %a, float %b) {
 ; MIPS32-NEXT:  $BB6_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB6_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB6_4: # %cond.end
 ; MIPS32-NEXT:    lw $1, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    mtc1 $1, $f0
@@ -325,6 +339,8 @@ define void @phi_ambiguous_float_in_gpr(i1 %cnd, float* %f32_ptr_a, float* %f32_
 ; MIPS32-NEXT:  $BB7_3: # %cond.false
 ; MIPS32-NEXT:    lw $1, 12($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    sw $1, 0($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    j $BB7_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB7_4: # %cond.end
 ; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
 ; MIPS32-NEXT:    lw $1, 0($sp) # 4-byte Folded Reload
@@ -372,6 +388,8 @@ define double @phi_double(double %a, double %b, i1 %cnd) {
 ; MIPS32-NEXT:  $BB8_3: # %cond.false
 ; MIPS32-NEXT:    ldc1 $f0, 16($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    sdc1 $f0, 0($sp) # 8-byte Folded Spill
+; MIPS32-NEXT:    j $BB8_4
+; MIPS32-NEXT:    nop
 ; MIPS32-NEXT:  $BB8_4: # %cond.end
 ; MIPS32-NEXT:    ldc1 $f0, 0($sp) # 8-byte Folded Reload
 ; MIPS32-NEXT:    addiu $sp, $sp, 24
diff --git a/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll b/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll
new file mode 100644
index 0000000..3873f7f
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/fallthrough-branch.ll
@@ -0,0 +1,48 @@
+; RUN: llc -O0 -stop-before=livedebugvalues < %s | FileCheck %s
+
+; ModuleID = '/tmp/t.o'
+source_filename = "/tmp/t.o"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx11.0.0"
+
+define swiftcc void @"$s1t1f1bySb_tF"(i1 %0) !dbg !35 {
+  %2 = alloca i1, align 8
+  %3 = bitcast i1* %2 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %3, i8 0, i64 1, i1 false)
+  store i1 %0, i1* %2, align 8, !dbg !37
+; CHECK:   B %bb.1, debug-location !{{[0-9]+}}
+  br i1 %0, label %4, label %5, !dbg !38
+
+4:                                                ; preds = %1
+; Check that at -O0 the branches and their debug locations are not eliminated.
+; CHECK:   B %bb.3, debug-location !{{[0-9]+}}
+  br label %6, !dbg !39
+
+5:                                                ; preds = %1
+; CHECK:   B %bb.3, debug-location !{{[0-9]+}}
+  br label %6, !dbg !40
+
+6:                                                ; preds = %4, %5
+  ret void, !dbg !39
+}
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #1
+attributes #1 = { argmemonly nofree nosync nounwind willreturn writeonly }
+
+!llvm.module.flags = !{!6, !7, !14}
+!llvm.dbg.cu = !{!15, !27}
+
+!6 = !{i32 7, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"Swift Version", i32 7}
+!15 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !16, producer: "Swift", emissionKind: LineTablesOnly)
+!16 = !DIFile(filename: "t.swift", directory: "/tmp")
+!17 = !{}
+!27 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !16, emissionKind: LineTablesOnly)
+!35 = distinct !DISubprogram(name: "f", linkageName: "$s1t1f1bySb_tF", scope: !15, file: !16, line: 1, type: !36, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !15, retainedNodes: !17)
+!36 = !DISubroutineType(types: null)
+!37 = !DILocation(line: 0, scope: !35)
+!38 = !DILocation(line: 2, column: 9, scope: !35)
+!39 = !DILocation(line: 3, column: 1, scope: !35)
+!40 = !DILocation(line: 2, column: 18, scope: !35)
-- 
2.7.4