Expand masked mem intrinsics correctly wrt big-endian

Need to take endianness into account when doing vector to scalar casts
such as %bc = bitcast <8 x i1> %v to i8
Companion commit for https://reviews.llvm.org/D94867
Upload in response to
https://lists.llvm.org/pipermail/llvm-dev/2021-January/147862.html
Attempting to document the actual memory layout rules for vectors in
https://reviews.llvm.org/D94964

Differential Revision: https://reviews.llvm.org/D94765

diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 11bd92c..8740b35 100644 (file)
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -105,6 +105,11 @@ static bool isConstantIntVector(Value *Mask) {
   return true;
 }
 
+static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
+                                unsigned Idx) {
+  return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
+}
+
 // Translate a masked load intrinsic like
 // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align,
 //                               <16 x i1> %mask, <16 x i32> %passthru)
@@ -137,8 +142,8 @@ static bool isConstantIntVector(Value *Mask) {
 //  %10 = extractelement <16 x i1> %mask, i32 2
 //  br i1 %10, label %cond.load4, label %else5
 //
-static void scalarizeMaskedLoad(CallInst *CI, DomTreeUpdater *DTU,
-                                bool &ModifiedDT) {
+static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
+                                DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
@@ -207,7 +212,8 @@ static void scalarizeMaskedLoad(CallInst *CI, DomTreeUpdater *DTU,
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -278,8 +284,8 @@ static void scalarizeMaskedLoad(CallInst *CI, DomTreeUpdater *DTU,
 //   store i32 %6, i32* %7
 //   br label %else2
 //   . . .
-static void scalarizeMaskedStore(CallInst *CI, DomTreeUpdater *DTU,
-                                 bool &ModifiedDT) {
+static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
+                                 DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
@@ -340,7 +346,8 @@ static void scalarizeMaskedStore(CallInst *CI, DomTreeUpdater *DTU,
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -405,8 +412,8 @@ static void scalarizeMaskedStore(CallInst *CI, DomTreeUpdater *DTU,
 // . . .
 // %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
 // ret <16 x i32> %Result
-static void scalarizeMaskedGather(CallInst *CI, DomTreeUpdater *DTU,
-                                  bool &ModifiedDT) {
+static void scalarizeMaskedGather(const DataLayout &DL, CallInst *CI,
+                                  DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptrs = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
@@ -461,7 +468,8 @@ static void scalarizeMaskedGather(CallInst *CI, DomTreeUpdater *DTU,
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -534,8 +542,8 @@ static void scalarizeMaskedGather(CallInst *CI, DomTreeUpdater *DTU,
 // store i32 %Elt1, i32* %Ptr1, align 4
 // br label %else2
 //   . . .
-static void scalarizeMaskedScatter(CallInst *CI, DomTreeUpdater *DTU,
-                                   bool &ModifiedDT) {
+static void scalarizeMaskedScatter(const DataLayout &DL, CallInst *CI,
+                                   DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptrs = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
@@ -587,7 +595,8 @@ static void scalarizeMaskedScatter(CallInst *CI, DomTreeUpdater *DTU,
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -623,8 +632,8 @@ static void scalarizeMaskedScatter(CallInst *CI, DomTreeUpdater *DTU,
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedExpandLoad(CallInst *CI, DomTreeUpdater *DTU,
-                                      bool &ModifiedDT) {
+static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
+                                      DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Mask = CI->getArgOperand(1);
   Value *PassThru = CI->getArgOperand(2);
@@ -692,7 +701,8 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, DomTreeUpdater *DTU,
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -749,7 +759,8 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, DomTreeUpdater *DTU,
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedCompressStore(CallInst *CI, DomTreeUpdater *DTU,
+static void scalarizeMaskedCompressStore(const DataLayout &DL, CallInst *CI,
+                                         DomTreeUpdater *DTU,
                                          bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
@@ -800,7 +811,8 @@ static void scalarizeMaskedCompressStore(CallInst *CI, DomTreeUpdater *DTU,
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -934,14 +946,14 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
               CI->getType(),
               cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
         return false;
-      scalarizeMaskedLoad(CI, DTU, ModifiedDT);
+      scalarizeMaskedLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_store:
       if (TTI.isLegalMaskedStore(
               CI->getArgOperand(0)->getType(),
               cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
         return false;
-      scalarizeMaskedStore(CI, DTU, ModifiedDT);
+      scalarizeMaskedStore(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_gather: {
       unsigned AlignmentInt =
@@ -951,7 +963,7 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
           DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), LoadTy);
       if (TTI.isLegalMaskedGather(LoadTy, Alignment))
         return false;
-      scalarizeMaskedGather(CI, DTU, ModifiedDT);
+      scalarizeMaskedGather(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_scatter: {
@@ -962,18 +974,18 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
           DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), StoreTy);
       if (TTI.isLegalMaskedScatter(StoreTy, Alignment))
         return false;
-      scalarizeMaskedScatter(CI, DTU, ModifiedDT);
+      scalarizeMaskedScatter(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_expandload:
       if (TTI.isLegalMaskedExpandLoad(CI->getType()))
         return false;
-      scalarizeMaskedExpandLoad(CI, DTU, ModifiedDT);
+      scalarizeMaskedExpandLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_compressstore:
       if (TTI.isLegalMaskedCompressStore(CI->getArgOperand(0)->getType()))
         return false;
-      scalarizeMaskedCompressStore(CI, DTU, ModifiedDT);
+      scalarizeMaskedCompressStore(DL, CI, DTU, ModifiedDT);
       return true;
     }
   }

diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index 7ec6a24..d5f4236 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -175,16 +175,16 @@ define void @foo_sext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q2
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB5_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB5_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q1[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
 ; CHECK-BE-NEXT:  .LBB5_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB5_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB5_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
@@ -217,12 +217,12 @@ define void @foo_sext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:    mvnne r4, #1
 ; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r4, #3
-; CHECK-BE-NEXT:    lsls r2, r4, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r4, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
 entry:
@@ -325,16 +325,16 @@ define void @foo_sext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q2
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB6_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB6_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q1[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
 ; CHECK-BE-NEXT:  .LBB6_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB6_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB6_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q2
@@ -367,14 +367,14 @@ define void @foo_sext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:    mvnne r4, #1
 ; CHECK-BE-NEXT:    bfi r4, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r4, #3
-; CHECK-BE-NEXT:    lsls r2, r4, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, r3, d0
-; CHECK-BE-NEXT:    strdne r3, r2, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r4, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, r2, d1
-; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi r2, r3, d0
+; CHECK-BE-NEXT:    strdmi r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, r2, d1
+; CHECK-BE-NEXT:    strdne r2, r1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r4, r5, r7, pc}
 entry:
@@ -471,16 +471,16 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB7_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB7_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q2[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q1, q2
 ; CHECK-BE-NEXT:  .LBB7_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB7_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB7_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
@@ -507,12 +507,12 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
@@ -611,16 +611,16 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:    bfi r3, lr, #0, #1
 ; CHECK-BE-NEXT:    @ implicit-def: $q1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    beq .LBB8_2
+; CHECK-BE-NEXT:    lsls r3, r3, #30
+; CHECK-BE-NEXT:    bpl .LBB8_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldr r3, [r2]
 ; CHECK-BE-NEXT:    vmov.32 q2[1], r3
 ; CHECK-BE-NEXT:    vrev64.32 q1, q2
 ; CHECK-BE-NEXT:  .LBB8_2: @ %else
 ; CHECK-BE-NEXT:    vrev64.32 q2, q0
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB8_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB8_4
 ; CHECK-BE-NEXT:  @ %bb.3: @ %cond.load1
 ; CHECK-BE-NEXT:    ldr r1, [r2, #4]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
@@ -647,14 +647,14 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r12, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, r3, d0
-; CHECK-BE-NEXT:    strdne r3, r2, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #30
 ; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, r2, d1
-; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    vmovmi r2, r3, d0
+; CHECK-BE-NEXT:    strdmi r3, r2, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, r2, d1
+; CHECK-BE-NEXT:    strdne r2, r1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
 entry:
@@ -936,17 +936,17 @@ define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%
 ; CHECK-BE-NEXT:    and r3, lr, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #3, #1
-; CHECK-BE-NEXT:    lsls r3, r1, #31
-; CHECK-BE-NEXT:    bne .LBB18_6
+; CHECK-BE-NEXT:    lsls r3, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB18_6
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r3, r1, #30
+; CHECK-BE-NEXT:    lsls r3, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB18_7
 ; CHECK-BE-NEXT:  .LBB18_2: @ %else2
-; CHECK-BE-NEXT:    lsls r3, r1, #29
+; CHECK-BE-NEXT:    lsls r3, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB18_8
 ; CHECK-BE-NEXT:  .LBB18_3: @ %else5
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB18_5
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB18_5
 ; CHECK-BE-NEXT:  .LBB18_4: @ %cond.load7
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vins.f16 s0, s4
@@ -971,34 +971,34 @@ define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    pop {r7, pc}
 ; CHECK-BE-NEXT:  .LBB18_6: @ %cond.load
 ; CHECK-BE-NEXT:    vldr.16 s0, [r2]
-; CHECK-BE-NEXT:    lsls r3, r1, #30
+; CHECK-BE-NEXT:    lsls r3, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB18_2
 ; CHECK-BE-NEXT:  .LBB18_7: @ %cond.load1
 ; CHECK-BE-NEXT:    vldr.16 s4, [r2, #2]
 ; CHECK-BE-NEXT:    vins.f16 s0, s4
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s1
 ; CHECK-BE-NEXT:    vins.f16 s1, s4
-; CHECK-BE-NEXT:    lsls r3, r1, #29
+; CHECK-BE-NEXT:    lsls r3, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB18_3
 ; CHECK-BE-NEXT:  .LBB18_8: @ %cond.load4
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
@@ -1006,8 +1006,8 @@ define void @foo_v4f32_v4f16(<4 x float> *%dest, <4 x i16> *%mask, <4 x half> *%
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s1
 ; CHECK-BE-NEXT:    vldr.16 s1, [r2, #4]
 ; CHECK-BE-NEXT:    vins.f16 s1, s4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB18_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB18_4
 ; CHECK-BE-NEXT:    b .LBB18_5
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2
@@ -1139,17 +1139,17 @@ define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4
 ; CHECK-BE-NEXT:    and r3, lr, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #3, #1
-; CHECK-BE-NEXT:    lsls r3, r1, #31
-; CHECK-BE-NEXT:    bne .LBB19_6
+; CHECK-BE-NEXT:    lsls r3, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB19_6
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r3, r1, #30
+; CHECK-BE-NEXT:    lsls r3, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB19_7
 ; CHECK-BE-NEXT:  .LBB19_2: @ %else2
-; CHECK-BE-NEXT:    lsls r3, r1, #29
+; CHECK-BE-NEXT:    lsls r3, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB19_8
 ; CHECK-BE-NEXT:  .LBB19_3: @ %else5
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB19_5
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB19_5
 ; CHECK-BE-NEXT:  .LBB19_4: @ %cond.load7
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vins.f16 s0, s4
@@ -1174,34 +1174,34 @@ define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    pop {r7, pc}
 ; CHECK-BE-NEXT:  .LBB19_6: @ %cond.load
 ; CHECK-BE-NEXT:    vldr.16 s0, [r2]
-; CHECK-BE-NEXT:    lsls r3, r1, #30
+; CHECK-BE-NEXT:    lsls r3, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB19_2
 ; CHECK-BE-NEXT:  .LBB19_7: @ %cond.load1
 ; CHECK-BE-NEXT:    vldr.16 s4, [r2, #2]
 ; CHECK-BE-NEXT:    vins.f16 s0, s4
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s1
 ; CHECK-BE-NEXT:    vins.f16 s1, s4
-; CHECK-BE-NEXT:    lsls r3, r1, #29
+; CHECK-BE-NEXT:    lsls r3, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB19_3
 ; CHECK-BE-NEXT:  .LBB19_8: @ %cond.load4
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
@@ -1209,8 +1209,8 @@ define void @foo_v4f32_v4f16_unaligned(<4 x float> *%dest, <4 x i16> *%mask, <4
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s1
 ; CHECK-BE-NEXT:    vldr.16 s1, [r2, #4]
 ; CHECK-BE-NEXT:    vins.f16 s1, s4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB19_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB19_4
 ; CHECK-BE-NEXT:    b .LBB19_5
 entry:
   %0 = load <4 x i16>, <4 x i16>* %mask, align 2

diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
index 933424a..9d5e341 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
@@ -103,22 +103,22 @@ define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align1_undef(<4 x i32> *%dest, <4
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovne.32 q1[3], r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
@@ -254,22 +254,22 @@ define arm_aapcs_vfpcc <4 x i32> @zext16_masked_v4i32_align1_undef(<4 x i16> *%d
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q0[3], r0
 ; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
@@ -412,22 +412,22 @@ define arm_aapcs_vfpcc <4 x i32> @sext16_masked_v4i32_align1_undef(<4 x i16> *%d
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.32 q0[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q0[3], r0
 ; CHECK-BE-NEXT:    vmovlb.s16 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
@@ -657,38 +657,38 @@ define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align1_undef(<8 x i16> *%dest, <8
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrhmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi.16 q1[0], r2
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
 ; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #14]
+; CHECK-BE-NEXT:    vmovne.16 q1[7], r0
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
@@ -1291,22 +1291,22 @@ define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align1_undef(<4 x float> *%dest
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne s4, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    ldrmi r2, [r0]
+; CHECK-BE-NEXT:    vmovmi s4, r2
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi s5, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
 ; CHECK-BE-NEXT:    vmovmi s6, r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi s7, r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrne r0, [r0, #12]
+; CHECK-BE-NEXT:    vmovne s7, r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
@@ -1587,29 +1587,29 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB45_10
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    bmi .LBB45_10
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bmi .LBB45_11
 ; CHECK-BE-NEXT:  .LBB45_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bmi .LBB45_12
 ; CHECK-BE-NEXT:  .LBB45_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bmi .LBB45_13
 ; CHECK-BE-NEXT:  .LBB45_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bmi .LBB45_14
 ; CHECK-BE-NEXT:  .LBB45_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB45_15
 ; CHECK-BE-NEXT:  .LBB45_6: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB45_16
 ; CHECK-BE-NEXT:  .LBB45_7: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB45_9
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB45_9
 ; CHECK-BE-NEXT:  .LBB45_8: @ %cond.load19
 ; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
 ; CHECK-BE-NEXT:    strh.w r0, [sp]
@@ -1624,7 +1624,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    ldrh r2, [r0]
 ; CHECK-BE-NEXT:    strh.w r2, [sp, #28]
 ; CHECK-BE-NEXT:    vldr.16 s4, [sp, #28]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bpl .LBB45_2
 ; CHECK-BE-NEXT:  .LBB45_11: @ %cond.load1
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #2]
@@ -1632,7 +1632,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #24]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bpl .LBB45_3
 ; CHECK-BE-NEXT:  .LBB45_12: @ %cond.load4
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #4]
@@ -1640,7 +1640,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #20]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bpl .LBB45_4
 ; CHECK-BE-NEXT:  .LBB45_13: @ %cond.load7
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #6]
@@ -1648,7 +1648,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #16]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bpl .LBB45_5
 ; CHECK-BE-NEXT:  .LBB45_14: @ %cond.load10
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #8]
@@ -1656,7 +1656,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #12]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB45_6
 ; CHECK-BE-NEXT:  .LBB45_15: @ %cond.load13
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #10]
@@ -1664,7 +1664,7 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #8]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB45_7
 ; CHECK-BE-NEXT:  .LBB45_16: @ %cond.load16
 ; CHECK-BE-NEXT:    ldrh r2, [r0, #12]
@@ -1672,8 +1672,8 @@ define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest,
 ; CHECK-BE-NEXT:    vldr.16 s0, [sp, #4]
 ; CHECK-BE-NEXT:    vmov r2, s0
 ; CHECK-BE-NEXT:    vmov.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB45_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB45_8
 ; CHECK-BE-NEXT:    b .LBB45_9
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -1823,8 +1823,8 @@ define arm_aapcs_vfpcc <2 x i64> @masked_v2i64_align4_zero(<2 x i64> *%dest, <2
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB49_2
+; CHECK-BE-NEXT:    lsls r2, r2, #30
+; CHECK-BE-NEXT:    bpl .LBB49_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    vldr d1, .LCPI49_0
 ; CHECK-BE-NEXT:    vldr d0, [r0]
@@ -1832,9 +1832,9 @@ define arm_aapcs_vfpcc <2 x i64> @masked_v2i64_align4_zero(<2 x i64> *%dest, <2
 ; CHECK-BE-NEXT:  .LBB49_2:
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB49_3: @ %else
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vldrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:    .p2align 3
@@ -1916,8 +1916,8 @@ define arm_aapcs_vfpcc <2 x double> @masked_v2f64_align4_zero(<2 x double> *%des
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r2, #3
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB50_2
+; CHECK-BE-NEXT:    lsls r2, r2, #30
+; CHECK-BE-NEXT:    bpl .LBB50_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    vldr d1, .LCPI50_0
 ; CHECK-BE-NEXT:    vldr d0, [r0]
@@ -1925,9 +1925,9 @@ define arm_aapcs_vfpcc <2 x double> @masked_v2f64_align4_zero(<2 x double> *%des
 ; CHECK-BE-NEXT:  .LBB50_2:
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB50_3: @ %else
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vldrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:    .p2align 3
@@ -2028,8 +2028,8 @@ define arm_aapcs_vfpcc <4 x i16> @anyext_v4i16_align1(<4 x i16> *%dest, <4 x i32
 ; CHECK-BE-NEXT:    and r2, r3, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    beq .LBB52_2
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bpl .LBB52_2
 ; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
 ; CHECK-BE-NEXT:    ldrh r2, [r0]
 ; CHECK-BE-NEXT:    vdup.32 q1, r12
@@ -2038,18 +2038,18 @@ define arm_aapcs_vfpcc <4 x i16> @anyext_v4i16_align1(<4 x i16> *%dest, <4 x i32
 ; CHECK-BE-NEXT:  .LBB52_2:
 ; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-BE-NEXT:  .LBB52_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
 ; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    ldrhne r0, [r0, #6]
+; CHECK-BE-NEXT:    vmovne.32 q1[3], r0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr

diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
index a2cc365..16e886c 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
@@ -80,22 +80,22 @@ define arm_aapcs_vfpcc void @masked_v4i32_align1(<4 x i32> *%dest, <4 x i32> %a)
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -283,38 +283,38 @@ define arm_aapcs_vfpcc void @masked_v8i16_align1(<8 x i16> *%dest, <8 x i16> %a)
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r2, q1[0]
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[0]
+; CHECK-BE-NEXT:    strhmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[1]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[2]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[3]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[4]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[5]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[6]
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q1[7]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne.u16 r1, q1[7]
+; CHECK-BE-NEXT:    strhne r1, [r0, #14]
 ; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -540,22 +540,22 @@ define arm_aapcs_vfpcc void @masked_v4f32_align1(<4 x float> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strne r1, [r0, #12]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -789,29 +789,29 @@ define arm_aapcs_vfpcc void @masked_v8f16_align1(<8 x half> *%dest, <8 x half> %
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
 ; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
 ; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB16_9
+; CHECK-BE-NEXT:    lsls r2, r2, #24
+; CHECK-BE-NEXT:    bmi .LBB16_9
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bmi .LBB16_10
 ; CHECK-BE-NEXT:  .LBB16_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bmi .LBB16_11
 ; CHECK-BE-NEXT:  .LBB16_3: @ %else4
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bmi .LBB16_12
 ; CHECK-BE-NEXT:  .LBB16_4: @ %else6
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bmi .LBB16_13
 ; CHECK-BE-NEXT:  .LBB16_5: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB16_14
 ; CHECK-BE-NEXT:  .LBB16_6: @ %else10
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB16_15
 ; CHECK-BE-NEXT:  .LBB16_7: @ %else12
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB16_16
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB16_16
 ; CHECK-BE-NEXT:  .LBB16_8: @ %else14
 ; CHECK-BE-NEXT:    add sp, #40
 ; CHECK-BE-NEXT:    bx lr
@@ -819,47 +819,47 @@ define arm_aapcs_vfpcc void @masked_v8f16_align1(<8 x half> *%dest, <8 x half> %
 ; CHECK-BE-NEXT:    vstr.16 s4, [sp, #28]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #28]
 ; CHECK-BE-NEXT:    strh r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #25
 ; CHECK-BE-NEXT:    bpl .LBB16_2
 ; CHECK-BE-NEXT:  .LBB16_10: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s4
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #24]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #24]
 ; CHECK-BE-NEXT:    strh r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #26
 ; CHECK-BE-NEXT:    bpl .LBB16_3
 ; CHECK-BE-NEXT:  .LBB16_11: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s5, [sp, #20]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #20]
 ; CHECK-BE-NEXT:    strh r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    lsls r2, r1, #27
 ; CHECK-BE-NEXT:    bpl .LBB16_4
 ; CHECK-BE-NEXT:  .LBB16_12: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s5
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #16]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #16]
 ; CHECK-BE-NEXT:    strh r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
+; CHECK-BE-NEXT:    lsls r2, r1, #28
 ; CHECK-BE-NEXT:    bpl .LBB16_5
 ; CHECK-BE-NEXT:  .LBB16_13: @ %cond.store7
 ; CHECK-BE-NEXT:    vstr.16 s6, [sp, #12]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
 ; CHECK-BE-NEXT:    strh r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB16_6
 ; CHECK-BE-NEXT:  .LBB16_14: @ %cond.store9
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s6
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #8]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    strh r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB16_7
 ; CHECK-BE-NEXT:  .LBB16_15: @ %cond.store11
 ; CHECK-BE-NEXT:    vstr.16 s7, [sp, #4]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
 ; CHECK-BE-NEXT:    strh r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB16_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB16_8
 ; CHECK-BE-NEXT:  .LBB16_16: @ %cond.store13
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp]
@@ -989,12 +989,12 @@ define arm_aapcs_vfpcc void @masked_v2i64(<2 x i64> *%dest, <2 x i64> %a) {
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r3, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -1060,12 +1060,12 @@ define arm_aapcs_vfpcc void @masked_v2f64(<2 x double> *%dest, <2 x double> %a,
 ; CHECK-BE-NEXT:    mvnne r3, #1
 ; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r3, #3
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne d0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r3, #30
 ; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
+; CHECK-BE-NEXT:    vstrmi d0, [r0]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -1193,22 +1193,22 @@ define arm_aapcs_vfpcc void @masked_v4i16_align1(<4 x i16> *%dest, <4 x i32> %a)
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    itt mi
+; CHECK-BE-NEXT:    vmovmi r2, s4
+; CHECK-BE-NEXT:    strhmi r2, [r0]
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s5
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    itt mi
 ; CHECK-BE-NEXT:    vmovmi r2, s6
 ; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strhmi r1, [r0, #6]
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    itt ne
+; CHECK-BE-NEXT:    vmovne r1, s7
+; CHECK-BE-NEXT:    strhne r1, [r0, #6]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -1351,33 +1351,33 @@ define arm_aapcs_vfpcc void @masked_v4f16_align4(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB25_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB25_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB25_6
 ; CHECK-BE-NEXT:  .LBB25_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB25_7
 ; CHECK-BE-NEXT:  .LBB25_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB25_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB25_8
 ; CHECK-BE-NEXT:  .LBB25_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB25_5: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB25_2
 ; CHECK-BE-NEXT:  .LBB25_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB25_3
 ; CHECK-BE-NEXT:  .LBB25_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB25_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB25_4
 ; CHECK-BE-NEXT:  .LBB25_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
@@ -1523,33 +1523,33 @@ define arm_aapcs_vfpcc void @masked_v4f16_align2(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB26_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB26_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB26_6
 ; CHECK-BE-NEXT:  .LBB26_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB26_7
 ; CHECK-BE-NEXT:  .LBB26_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB26_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB26_8
 ; CHECK-BE-NEXT:  .LBB26_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 ; CHECK-BE-NEXT:  .LBB26_5: @ %cond.store
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB26_2
 ; CHECK-BE-NEXT:  .LBB26_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB26_3
 ; CHECK-BE-NEXT:  .LBB26_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB26_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB26_4
 ; CHECK-BE-NEXT:  .LBB26_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
@@ -1703,17 +1703,17 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
 ; CHECK-BE-NEXT:    vcvtb.f16.f32 s1, s6
 ; CHECK-BE-NEXT:    vcvtt.f16.f32 s1, s7
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    bne .LBB27_5
+; CHECK-BE-NEXT:    lsls r2, r1, #28
+; CHECK-BE-NEXT:    bmi .LBB27_5
 ; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bmi .LBB27_6
 ; CHECK-BE-NEXT:  .LBB27_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bmi .LBB27_7
 ; CHECK-BE-NEXT:  .LBB27_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB27_8
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    bne .LBB27_8
 ; CHECK-BE-NEXT:  .LBB27_4: @ %else6
 ; CHECK-BE-NEXT:    add sp, #20
 ; CHECK-BE-NEXT:    bx lr
@@ -1721,21 +1721,21 @@ define arm_aapcs_vfpcc void @masked_v4f16_align1(<4 x half> *%dest, <4 x float>
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp, #12]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
 ; CHECK-BE-NEXT:    strh r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
+; CHECK-BE-NEXT:    lsls r2, r1, #29
 ; CHECK-BE-NEXT:    bpl .LBB27_2
 ; CHECK-BE-NEXT:  .LBB27_6: @ %cond.store1
 ; CHECK-BE-NEXT:    vmovx.f16 s4, s0
 ; CHECK-BE-NEXT:    vstr.16 s4, [sp, #8]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
 ; CHECK-BE-NEXT:    strh r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
+; CHECK-BE-NEXT:    lsls r2, r1, #30
 ; CHECK-BE-NEXT:    bpl .LBB27_3
 ; CHECK-BE-NEXT:  .LBB27_7: @ %cond.store3
 ; CHECK-BE-NEXT:    vstr.16 s1, [sp, #4]
 ; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
 ; CHECK-BE-NEXT:    strh r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB27_4
+; CHECK-BE-NEXT:    lsls r1, r1, #31
+; CHECK-BE-NEXT:    beq .LBB27_4
 ; CHECK-BE-NEXT:  .LBB27_8: @ %cond.store5
 ; CHECK-BE-NEXT:    vmovx.f16 s0, s1
 ; CHECK-BE-NEXT:    vstr.16 s0, [sp]

diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
index 580c403..1299201 100644 (file)
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
@@ -1,32 +1,58 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve  | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64_be-linux-gnu -data-layout="E-m:o-i64:64-i128:128-n32:64-S128" | FileCheck -check-prefixes=CHECK,CHECK-BE %s
 
 define <2 x i64> @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %passthru) {
-; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i64(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i64(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP4]], align 8
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP9]], align 8
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)
   ret <2 x i64> %ret
@@ -64,30 +90,55 @@ define <2 x i64> @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %passthru)
 
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %passthru) {
-; CHECK-LABEL: @scalarize_v2i24(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i24(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i24(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i24, i24* [[TMP4]], align 1
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i24, i24* [[TMP9]], align 1
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>* %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)
   ret <2 x i24> %ret
@@ -95,30 +146,55 @@ define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %pass
 
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i48> @scalarize_v2i48(<2 x i48>* %p, <2 x i1> %mask, <2 x i48> %passthru) {
-; CHECK-LABEL: @scalarize_v2i48(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+; CHECK-LE-LABEL: @scalarize_v2i48(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.load:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.load1:
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-LE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-LE-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+;
+; CHECK-BE-LABEL: @scalarize_v2i48(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.load:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = load i48, i48* [[TMP4]], align 2
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP5]], i64 0
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP6]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = icmp ne i2 [[TMP7]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP8]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.load1:
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    [[TMP10:%.*]] = load i48, i48* [[TMP9]], align 2
+; CHECK-BE-NEXT:    [[TMP11:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP10]], i64 1
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP11]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-BE-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
 ;
   %ret = call <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>* %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)
   ret <2 x i48> %ret

diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
index e3245ca..c729a2f 100644 (file)
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
@@ -1,30 +1,54 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck %s
-; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck -check-prefixes=CHECK,CHECK-LE %s
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=aarch64_be-linux-gnu -data-layout="E-m:o-i64:64-i128:128-n32:64-S128" | FileCheck -check-prefixes=CHECK,CHECK-BE %s
 
 define void @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %data) {
-; CHECK-LABEL: @scalarize_v2i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
-; CHECK-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
-; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
-; CHECK-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
-; CHECK-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    ret void
+; CHECK-LE-LABEL: @scalarize_v2i64(
+; CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-LE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-LE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-LE:       cond.store:
+; CHECK-LE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-LE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-LE-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
+; CHECK-LE-NEXT:    br label [[ELSE]]
+; CHECK-LE:       else:
+; CHECK-LE-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-LE-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-LE-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-LE:       cond.store1:
+; CHECK-LE-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-LE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-LE-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
+; CHECK-LE-NEXT:    br label [[ELSE2]]
+; CHECK-LE:       else2:
+; CHECK-LE-NEXT:    ret void
+;
+; CHECK-BE-LABEL: @scalarize_v2i64(
+; CHECK-BE-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-BE-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-BE-NEXT:    [[TMP2:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-BE-NEXT:    [[TMP3:%.*]] = icmp ne i2 [[TMP2]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP3]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-BE:       cond.store:
+; CHECK-BE-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-BE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-BE-NEXT:    store i64 [[TMP4]], i64* [[TMP5]], align 8
+; CHECK-BE-NEXT:    br label [[ELSE]]
+; CHECK-BE:       else:
+; CHECK-BE-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-BE-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-BE-NEXT:    br i1 [[TMP7]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK-BE:       cond.store1:
+; CHECK-BE-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-BE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-BE-NEXT:    store i64 [[TMP8]], i64* [[TMP9]], align 8
+; CHECK-BE-NEXT:    br label [[ELSE2]]
+; CHECK-BE:       else2:
+; CHECK-BE-NEXT:    ret void
 ;
   call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 128, <2 x i1> %mask)
   ret void