From 369cd2ae5205e880e2db209070abadf459289322 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 4 May 2021 04:52:28 -0700 Subject: [PATCH] Revert "[SLP]Allow masked gathers only if allowed by target." This reverts commit fd18547e0721983dcb273670d16341921f831e50. Need to add a check for the size of the vectorization tree to avoid some extra vectorization. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 28 +- .../SLPVectorizer/X86/pr47629-inseltpoison.ll | 442 ++++++++++++++------- llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll | 442 ++++++++++++++------- .../test/Transforms/SLPVectorizer/X86/tiny-tree.ll | 24 +- 4 files changed, 603 insertions(+), 333 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 5c3ee62..5e2a8a1 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3921,31 +3921,9 @@ InstructionCost BoUpSLP::getEntryCost(TreeEntry *E) { CostKind, VL0); } else { assert(E->State == TreeEntry::ScatterVectorize && "Unknown EntryState"); - if (TTI->isLegalMaskedGather(VecTy, alignment)) { - VecLdCost = TTI->getGatherScatterOpCost( - Instruction::Load, VecTy, - cast(VL0)->getPointerOperand(), - /*VariableMask=*/false, alignment, CostKind, VL0); - } else { - // Lower just to a gather if masked gather is not legal. Also, - // compensate the cost of next entry for pointers. - VecLdCost = - getGatherCost(VL); - // Tru to compensate the cost of the next entry for pointers iff all - // users are ScatterVectorize nodes. - const auto *It = find_if( - VectorizableTree, [E](const std::unique_ptr &TE) { - return !TE->UserTreeIndices.empty() && - all_of(TE->UserTreeIndices, - [](const EdgeInfo &EI) { - return EI.UserTE->State == - TreeEntry::ScatterVectorize; - }) && - TE->UserTreeIndices.front().UserTE == E; - }); - if (It != VectorizableTree.end()) - VecLdCost -= getEntryCost(It->get()); - } + VecLdCost = TTI->getGatherScatterOpCost( + Instruction::Load, VecTy, cast(VL0)->getPointerOperand(), + /*VariableMask=*/false, alignment, CostKind, VL0); } if (!NeedToShuffleReuses && !E->ReorderIndices.empty()) { SmallVector NewMask; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll index a9ee98b..7564d3e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll @@ -8,19 +8,19 @@ define void @gather_load(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { ; CHECK-LABEL: @gather_load( ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 -; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP1]], align 4, [[TBAA0:!tbaa !.*]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 -; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4, [[TBAA0]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, !tbaa [[TBAA0]] -; CHECK-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP3]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; CHECK-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0]] ; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0 ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i32 1 ; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 2 ; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 3 ; CHECK-NEXT: [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], ; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* -; CHECK-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP15]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP15]], align 4, [[TBAA0]] ; CHECK-NEXT: ret void ; %3 = getelementptr inbounds i32, i32* %1, i64 1 @@ -44,15 +44,69 @@ define void @gather_load(i32* noalias nocapture %0, i32* noalias nocapture reado } define void @gather_load_2(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { -; CHECK-LABEL: @gather_load_2( -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> -; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], -; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* -; CHECK-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, !tbaa [[TBAA0]] -; CHECK-NEXT: ret void +; SSE-LABEL: @gather_load_2( +; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 +; SSE-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]] +; SSE-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1 +; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; SSE-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10 +; SSE-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2 +; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; SSE-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3 +; SSE-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3 +; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; SSE-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5 +; SSE-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4 +; SSE-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]] +; SSE-NEXT: ret void +; +; AVX-LABEL: @gather_load_2( +; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 +; AVX-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]] +; AVX-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1 +; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10 +; AVX-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2 +; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; AVX-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3 +; AVX-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3 +; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; AVX-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5 +; AVX-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4 +; AVX-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]] +; AVX-NEXT: ret void +; +; AVX2-LABEL: @gather_load_2( +; AVX2-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 +; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> +; AVX2-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0:!tbaa !.*]] +; AVX2-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], +; AVX2-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]] +; AVX2-NEXT: ret void +; +; AVX512-LABEL: @gather_load_2( +; AVX512-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 +; AVX512-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX512-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> +; AVX512-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0:!tbaa !.*]] +; AVX512-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], +; AVX512-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* +; AVX512-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]] +; AVX512-NEXT: ret void ; %3 = getelementptr inbounds i32, i32* %1, i64 1 %4 = load i32, i32* %3, align 4, !tbaa !2 @@ -79,97 +133,143 @@ define void @gather_load_2(i32* noalias nocapture %0, i32* noalias nocapture rea define void @gather_load_3(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { ; SSE-LABEL: @gather_load_3( -; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; SSE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 -; SSE-NEXT: [[TMP7:%.*]] = insertelement <4 x i32*> [[TMP6]], i32* [[TMP3]], i32 1 -; SSE-NEXT: [[TMP8:%.*]] = insertelement <4 x i32*> [[TMP7]], i32* [[TMP4]], i32 2 -; SSE-NEXT: [[TMP9:%.*]] = insertelement <4 x i32*> [[TMP8]], i32* [[TMP5]], i32 3 -; SSE-NEXT: [[TMP10:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP9]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP10]], -; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 4 -; SSE-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP11]], <4 x i32>* [[TMP13]], align 4, !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP15:%.*]] = getelementptr i32, <4 x i32*> [[TMP14]], <4 x i64> -; SSE-NEXT: [[TMP16:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP15]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP17:%.*]] = add <4 x i32> [[TMP16]], -; SSE-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP18]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; SSE-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 +; SSE-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2 +; SSE-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; SSE-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 +; SSE-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3 +; SSE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; SSE-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 +; SSE-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4 +; SSE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4 +; SSE-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 +; SSE-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1 +; SSE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; SSE-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; SSE-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2 +; SSE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; SSE-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; SSE-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3 +; SSE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; SSE-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; SSE-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4 +; SSE-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_3( -; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; AVX-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 -; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i32*> poison, i32* [[TMP1]], i32 0 -; AVX-NEXT: [[TMP11:%.*]] = insertelement <8 x i32*> [[TMP10]], i32* [[TMP3]], i32 1 -; AVX-NEXT: [[TMP12:%.*]] = insertelement <8 x i32*> [[TMP11]], i32* [[TMP4]], i32 2 -; AVX-NEXT: [[TMP13:%.*]] = insertelement <8 x i32*> [[TMP12]], i32* [[TMP5]], i32 3 -; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i32*> [[TMP13]], i32* [[TMP6]], i32 4 -; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x i32*> [[TMP14]], i32* [[TMP7]], i32 5 -; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x i32*> [[TMP15]], i32* [[TMP8]], i32 6 -; AVX-NEXT: [[TMP17:%.*]] = insertelement <8 x i32*> [[TMP16]], i32* [[TMP9]], i32 7 -; AVX-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP17]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP18]], -; AVX-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP0:%.*]] to <8 x i32>* -; AVX-NEXT: store <8 x i32> [[TMP19]], <8 x i32>* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 +; AVX-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2 +; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; AVX-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 +; AVX-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3 +; AVX-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; AVX-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 +; AVX-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4 +; AVX-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4 +; AVX-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 +; AVX-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1 +; AVX-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; AVX-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; AVX-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2 +; AVX-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; AVX-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; AVX-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3 +; AVX-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; AVX-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; AVX-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4 +; AVX-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_3( -; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; AVX2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 -; AVX2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX2-NEXT: [[TMP10:%.*]] = insertelement <8 x i32*> poison, i32* [[TMP1]], i32 0 -; AVX2-NEXT: [[TMP11:%.*]] = insertelement <8 x i32*> [[TMP10]], i32* [[TMP3]], i32 1 -; AVX2-NEXT: [[TMP12:%.*]] = insertelement <8 x i32*> [[TMP11]], i32* [[TMP4]], i32 2 -; AVX2-NEXT: [[TMP13:%.*]] = insertelement <8 x i32*> [[TMP12]], i32* [[TMP5]], i32 3 -; AVX2-NEXT: [[TMP14:%.*]] = insertelement <8 x i32*> [[TMP13]], i32* [[TMP6]], i32 4 -; AVX2-NEXT: [[TMP15:%.*]] = insertelement <8 x i32*> [[TMP14]], i32* [[TMP7]], i32 5 -; AVX2-NEXT: [[TMP16:%.*]] = insertelement <8 x i32*> [[TMP15]], i32* [[TMP8]], i32 6 -; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x i32*> [[TMP16]], i32* [[TMP9]], i32 7 -; AVX2-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP17]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX2-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP18]], -; AVX2-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP0:%.*]] to <8 x i32>* -; AVX2-NEXT: store <8 x i32> [[TMP19]], <8 x i32>* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX2-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 +; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> +; AVX2-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX2-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], +; AVX2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; AVX2-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; AVX2-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2 +; AVX2-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; AVX2-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; AVX2-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3 +; AVX2-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; AVX2-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; AVX2-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4 +; AVX2-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_3( -; AVX512-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 ; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 -; AVX512-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 ; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer ; AVX512-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> -; AVX512-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], ; AVX512-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 ; AVX512-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* -; AVX512-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX512-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2 ; AVX512-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 -; AVX512-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX512-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3 ; AVX512-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 -; AVX512-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX512-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4 -; AVX512-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %3 = load i32, i32* %1, align 4, !tbaa !2 @@ -215,68 +315,114 @@ define void @gather_load_3(i32* noalias nocapture %0, i32* noalias nocapture rea define void @gather_load_4(i32* noalias nocapture %t0, i32* noalias nocapture readonly %t1) { ; SSE-LABEL: @gather_load_4( +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 ; SSE-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 +; SSE-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2 ; SSE-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 +; SSE-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3 ; SSE-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 -; SSE-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 4 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> poison, i32* [[T1]], i32 0 -; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> -; SSE-NEXT: [[TMP4:%.*]] = insertelement <4 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; SSE-NEXT: [[TMP5:%.*]] = insertelement <4 x i32*> [[TMP4]], i32* [[T10]], i32 2 -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> [[TMP5]], i32* [[T14]], i32 3 -; SSE-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP8:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP7]], -; SSE-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP8]], -; SSE-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP12:%.*]] = bitcast i32* [[T17]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4 +; SSE-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; SSE-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 +; SSE-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; SSE-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 +; SSE-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; SSE-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 +; SSE-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 +; SSE-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; SSE-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]] +; SSE-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]] +; SSE-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]] +; SSE-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]] +; SSE-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; SSE-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; SSE-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; SSE-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; SSE-NEXT: [[T8:%.*]] = add i32 [[T7]], 2 +; SSE-NEXT: [[T12:%.*]] = add i32 [[T11]], 3 +; SSE-NEXT: [[T16:%.*]] = add i32 [[T15]], 4 +; SSE-NEXT: [[T20:%.*]] = add i32 [[T19]], 1 +; SSE-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; SSE-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; SSE-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; SSE-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_4( +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 ; AVX-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 +; AVX-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2 ; AVX-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 +; AVX-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3 ; AVX-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 +; AVX-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4 ; AVX-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; AVX-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 ; AVX-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; AVX-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 ; AVX-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; AVX-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX-NEXT: [[TMP1:%.*]] = insertelement <8 x i32*> poison, i32* [[T1]], i32 0 -; AVX-NEXT: [[TMP2:%.*]] = insertelement <8 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; AVX-NEXT: [[TMP3:%.*]] = insertelement <8 x i32*> [[TMP2]], i32* [[T10]], i32 2 -; AVX-NEXT: [[TMP4:%.*]] = insertelement <8 x i32*> [[TMP3]], i32* [[T14]], i32 3 -; AVX-NEXT: [[TMP5:%.*]] = insertelement <8 x i32*> [[TMP4]], i32* [[T18]], i32 4 -; AVX-NEXT: [[TMP6:%.*]] = insertelement <8 x i32*> [[TMP5]], i32* [[T22]], i32 5 -; AVX-NEXT: [[TMP7:%.*]] = insertelement <8 x i32*> [[TMP6]], i32* [[T26]], i32 6 -; AVX-NEXT: [[TMP8:%.*]] = insertelement <8 x i32*> [[TMP7]], i32* [[T30]], i32 7 -; AVX-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP8]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX-NEXT: [[TMP10:%.*]] = add <8 x i32> [[TMP9]], -; AVX-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0:%.*]] to <8 x i32>* -; AVX-NEXT: store <8 x i32> [[TMP10]], <8 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]] +; AVX-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]] +; AVX-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]] +; AVX-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]] +; AVX-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; AVX-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; AVX-NEXT: [[T8:%.*]] = add i32 [[T7]], 2 +; AVX-NEXT: [[T12:%.*]] = add i32 [[T11]], 3 +; AVX-NEXT: [[T16:%.*]] = add i32 [[T15]], 4 +; AVX-NEXT: [[T20:%.*]] = add i32 [[T19]], 1 +; AVX-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; AVX-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; AVX-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; AVX-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_4( -; AVX2-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 -; AVX2-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 -; AVX2-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 -; AVX2-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; AVX2-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 +; AVX2-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> poison, i32* [[T1:%.*]], i32 0 +; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> +; AVX2-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 ; AVX2-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; AVX2-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 ; AVX2-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; AVX2-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX2-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX2-NEXT: [[TMP1:%.*]] = insertelement <8 x i32*> poison, i32* [[T1]], i32 0 -; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; AVX2-NEXT: [[TMP3:%.*]] = insertelement <8 x i32*> [[TMP2]], i32* [[T10]], i32 2 -; AVX2-NEXT: [[TMP4:%.*]] = insertelement <8 x i32*> [[TMP3]], i32* [[T14]], i32 3 -; AVX2-NEXT: [[TMP5:%.*]] = insertelement <8 x i32*> [[TMP4]], i32* [[T18]], i32 4 -; AVX2-NEXT: [[TMP6:%.*]] = insertelement <8 x i32*> [[TMP5]], i32* [[T22]], i32 5 -; AVX2-NEXT: [[TMP7:%.*]] = insertelement <8 x i32*> [[TMP6]], i32* [[T26]], i32 6 -; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i32*> [[TMP7]], i32* [[T30]], i32 7 -; AVX2-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP8]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX2-NEXT: [[TMP10:%.*]] = add <8 x i32> [[TMP9]], -; AVX2-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0:%.*]] to <8 x i32>* -; AVX2-NEXT: store <8 x i32> [[TMP10]], <8 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX2-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; AVX2-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], +; AVX2-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; AVX2-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; AVX2-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; AVX2-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_4( @@ -290,22 +436,22 @@ define void @gather_load_4(i32* noalias nocapture %t0, i32* noalias nocapture re ; AVX512-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 ; AVX512-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX512-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX512-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; AVX512-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX512-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX512-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX512-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX512-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] ; AVX512-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 ; AVX512-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], ; AVX512-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 ; AVX512-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 ; AVX512-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 -; AVX512-NEXT: store i32 [[T4]], i32* [[T0]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>* -; AVX512-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T24]], i32* [[T21]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T28]], i32* [[T25]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T32]], i32* [[T29]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %t5 = getelementptr inbounds i32, i32* %t0, i64 1 @@ -363,21 +509,21 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; SSE-NEXT: [[TMP7:%.*]] = insertelement <4 x float*> [[TMP6]], float* [[TMP3]], i32 1 ; SSE-NEXT: [[TMP8:%.*]] = insertelement <4 x float*> [[TMP7]], float* [[TMP4]], i32 2 ; SSE-NEXT: [[TMP9:%.*]] = insertelement <4 x float*> [[TMP8]], float* [[TMP5]], i32 3 -; SSE-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP9]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP9]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP11:%.*]] = shufflevector <4 x float*> [[TMP6]], <4 x float*> undef, <4 x i32> zeroinitializer ; SSE-NEXT: [[TMP12:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP12]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP12]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP14:%.*]] = fdiv <4 x float> [[TMP10]], [[TMP13]] ; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, float* [[TMP0:%.*]], i64 4 ; SSE-NEXT: [[TMP16:%.*]] = bitcast float* [[TMP0]] to <4 x float>* -; SSE-NEXT: store <4 x float> [[TMP14]], <4 x float>* [[TMP16]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: store <4 x float> [[TMP14]], <4 x float>* [[TMP16]], align 4, [[TBAA0]] ; SSE-NEXT: [[TMP17:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP17]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP17]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP19:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP20:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP19]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP20:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP19]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP21:%.*]] = fdiv <4 x float> [[TMP18]], [[TMP20]] ; SSE-NEXT: [[TMP22:%.*]] = bitcast float* [[TMP15]] to <4 x float>* -; SSE-NEXT: store <4 x float> [[TMP21]], <4 x float>* [[TMP22]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: store <4 x float> [[TMP21]], <4 x float>* [[TMP22]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_div( @@ -396,13 +542,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_div( @@ -421,13 +567,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX2-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX2-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX2-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX2-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX2-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX2-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX2-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX2-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX2-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_div( @@ -446,13 +592,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX512-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX512-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX512-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX512-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX512-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX512-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX512-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX512-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %3 = load float, float* %1, align 4, !tbaa !2 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll index fdf023d..3aae7ba 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll @@ -8,19 +8,19 @@ define void @gather_load(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { ; CHECK-LABEL: @gather_load( ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 -; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP1]], align 4, [[TBAA0:!tbaa !.*]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 -; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4, [[TBAA0]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, !tbaa [[TBAA0]] -; CHECK-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP3]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; CHECK-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0]] ; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0 ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i32 1 ; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 2 ; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 3 ; CHECK-NEXT: [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], ; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* -; CHECK-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP15]], align 4, !tbaa [[TBAA0]] +; CHECK-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP15]], align 4, [[TBAA0]] ; CHECK-NEXT: ret void ; %3 = getelementptr inbounds i32, i32* %1, i64 1 @@ -44,15 +44,69 @@ define void @gather_load(i32* noalias nocapture %0, i32* noalias nocapture reado } define void @gather_load_2(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { -; CHECK-LABEL: @gather_load_2( -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> -; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], -; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* -; CHECK-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, !tbaa [[TBAA0]] -; CHECK-NEXT: ret void +; SSE-LABEL: @gather_load_2( +; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 +; SSE-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]] +; SSE-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1 +; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; SSE-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10 +; SSE-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2 +; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; SSE-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3 +; SSE-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3 +; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; SSE-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5 +; SSE-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4 +; SSE-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]] +; SSE-NEXT: ret void +; +; AVX-LABEL: @gather_load_2( +; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1 +; AVX-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]] +; AVX-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1 +; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10 +; AVX-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2 +; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; AVX-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3 +; AVX-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3 +; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; AVX-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5 +; AVX-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4 +; AVX-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]] +; AVX-NEXT: ret void +; +; AVX2-LABEL: @gather_load_2( +; AVX2-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 +; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> +; AVX2-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0:!tbaa !.*]] +; AVX2-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], +; AVX2-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]] +; AVX2-NEXT: ret void +; +; AVX512-LABEL: @gather_load_2( +; AVX512-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1:%.*]], i32 0 +; AVX512-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX512-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> +; AVX512-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0:!tbaa !.*]] +; AVX512-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], +; AVX512-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>* +; AVX512-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]] +; AVX512-NEXT: ret void ; %3 = getelementptr inbounds i32, i32* %1, i64 1 %4 = load i32, i32* %3, align 4, !tbaa !2 @@ -79,97 +133,143 @@ define void @gather_load_2(i32* noalias nocapture %0, i32* noalias nocapture rea define void @gather_load_3(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) { ; SSE-LABEL: @gather_load_3( -; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; SSE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 -; SSE-NEXT: [[TMP7:%.*]] = insertelement <4 x i32*> [[TMP6]], i32* [[TMP3]], i32 1 -; SSE-NEXT: [[TMP8:%.*]] = insertelement <4 x i32*> [[TMP7]], i32* [[TMP4]], i32 2 -; SSE-NEXT: [[TMP9:%.*]] = insertelement <4 x i32*> [[TMP8]], i32* [[TMP5]], i32 3 -; SSE-NEXT: [[TMP10:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP9]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP10]], -; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 4 -; SSE-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP11]], <4 x i32>* [[TMP13]], align 4, !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP15:%.*]] = getelementptr i32, <4 x i32*> [[TMP14]], <4 x i64> -; SSE-NEXT: [[TMP16:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP15]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP17:%.*]] = add <4 x i32> [[TMP16]], -; SSE-NEXT: [[TMP18:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP17]], <4 x i32>* [[TMP18]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; SSE-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 +; SSE-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2 +; SSE-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; SSE-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 +; SSE-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3 +; SSE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; SSE-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 +; SSE-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4 +; SSE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4 +; SSE-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 +; SSE-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1 +; SSE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; SSE-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; SSE-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2 +; SSE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; SSE-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; SSE-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3 +; SSE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; SSE-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; SSE-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]] +; SSE-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4 +; SSE-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_3( -; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; AVX-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 -; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i32*> poison, i32* [[TMP1]], i32 0 -; AVX-NEXT: [[TMP11:%.*]] = insertelement <8 x i32*> [[TMP10]], i32* [[TMP3]], i32 1 -; AVX-NEXT: [[TMP12:%.*]] = insertelement <8 x i32*> [[TMP11]], i32* [[TMP4]], i32 2 -; AVX-NEXT: [[TMP13:%.*]] = insertelement <8 x i32*> [[TMP12]], i32* [[TMP5]], i32 3 -; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i32*> [[TMP13]], i32* [[TMP6]], i32 4 -; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x i32*> [[TMP14]], i32* [[TMP7]], i32 5 -; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x i32*> [[TMP15]], i32* [[TMP8]], i32 6 -; AVX-NEXT: [[TMP17:%.*]] = insertelement <8 x i32*> [[TMP16]], i32* [[TMP9]], i32 7 -; AVX-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP17]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP18]], -; AVX-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP0:%.*]] to <8 x i32>* -; AVX-NEXT: store <8 x i32> [[TMP19]], <8 x i32>* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11 +; AVX-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2 +; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2 +; AVX-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 +; AVX-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3 +; AVX-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3 +; AVX-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 +; AVX-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4 +; AVX-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4 +; AVX-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 +; AVX-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1 +; AVX-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; AVX-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; AVX-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2 +; AVX-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; AVX-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; AVX-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3 +; AVX-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; AVX-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; AVX-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]] +; AVX-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4 +; AVX-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_3( -; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 11 -; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4 -; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15 -; AVX2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18 -; AVX2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX2-NEXT: [[TMP10:%.*]] = insertelement <8 x i32*> poison, i32* [[TMP1]], i32 0 -; AVX2-NEXT: [[TMP11:%.*]] = insertelement <8 x i32*> [[TMP10]], i32* [[TMP3]], i32 1 -; AVX2-NEXT: [[TMP12:%.*]] = insertelement <8 x i32*> [[TMP11]], i32* [[TMP4]], i32 2 -; AVX2-NEXT: [[TMP13:%.*]] = insertelement <8 x i32*> [[TMP12]], i32* [[TMP5]], i32 3 -; AVX2-NEXT: [[TMP14:%.*]] = insertelement <8 x i32*> [[TMP13]], i32* [[TMP6]], i32 4 -; AVX2-NEXT: [[TMP15:%.*]] = insertelement <8 x i32*> [[TMP14]], i32* [[TMP7]], i32 5 -; AVX2-NEXT: [[TMP16:%.*]] = insertelement <8 x i32*> [[TMP15]], i32* [[TMP8]], i32 6 -; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x i32*> [[TMP16]], i32* [[TMP9]], i32 7 -; AVX2-NEXT: [[TMP18:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP17]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX2-NEXT: [[TMP19:%.*]] = add <8 x i32> [[TMP18]], -; AVX2-NEXT: [[TMP20:%.*]] = bitcast i32* [[TMP0:%.*]] to <8 x i32>* -; AVX2-NEXT: store <8 x i32> [[TMP19]], <8 x i32>* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 +; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 +; AVX2-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 +; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> +; AVX2-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX2-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], +; AVX2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 +; AVX2-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 +; AVX2-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2 +; AVX2-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 +; AVX2-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 +; AVX2-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3 +; AVX2-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 +; AVX2-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 +; AVX2-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4 +; AVX2-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_3( -; AVX512-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1 ; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1 -; AVX512-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> poison, i32* [[TMP1]], i32 0 ; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer ; AVX512-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> -; AVX512-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], ; AVX512-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5 ; AVX512-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>* -; AVX512-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9 -; AVX512-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2 ; AVX512-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6 -; AVX512-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6 -; AVX512-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3 ; AVX512-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7 -; AVX512-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21 -; AVX512-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4 -; AVX512-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %3 = load i32, i32* %1, align 4, !tbaa !2 @@ -215,68 +315,114 @@ define void @gather_load_3(i32* noalias nocapture %0, i32* noalias nocapture rea define void @gather_load_4(i32* noalias nocapture %t0, i32* noalias nocapture readonly %t1) { ; SSE-LABEL: @gather_load_4( +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 ; SSE-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 +; SSE-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2 ; SSE-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 +; SSE-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3 ; SSE-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 -; SSE-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 4 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> poison, i32* [[T1]], i32 0 -; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> -; SSE-NEXT: [[TMP4:%.*]] = insertelement <4 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; SSE-NEXT: [[TMP5:%.*]] = insertelement <4 x i32*> [[TMP4]], i32* [[T10]], i32 2 -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> [[TMP5]], i32* [[T14]], i32 3 -; SSE-NEXT: [[TMP7:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP6]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP8:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP7]], -; SSE-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP8]], -; SSE-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] -; SSE-NEXT: [[TMP12:%.*]] = bitcast i32* [[T17]] to <4 x i32>* -; SSE-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4 +; SSE-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; SSE-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 +; SSE-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; SSE-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 +; SSE-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; SSE-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 +; SSE-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 +; SSE-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; SSE-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]] +; SSE-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]] +; SSE-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]] +; SSE-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]] +; SSE-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; SSE-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; SSE-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; SSE-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; SSE-NEXT: [[T8:%.*]] = add i32 [[T7]], 2 +; SSE-NEXT: [[T12:%.*]] = add i32 [[T11]], 3 +; SSE-NEXT: [[T16:%.*]] = add i32 [[T15]], 4 +; SSE-NEXT: [[T20:%.*]] = add i32 [[T19]], 1 +; SSE-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; SSE-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; SSE-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; SSE-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; SSE-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_4( +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 ; AVX-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 +; AVX-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2 ; AVX-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 +; AVX-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3 ; AVX-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 +; AVX-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4 ; AVX-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; AVX-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 ; AVX-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; AVX-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 ; AVX-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; AVX-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX-NEXT: [[TMP1:%.*]] = insertelement <8 x i32*> poison, i32* [[T1]], i32 0 -; AVX-NEXT: [[TMP2:%.*]] = insertelement <8 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; AVX-NEXT: [[TMP3:%.*]] = insertelement <8 x i32*> [[TMP2]], i32* [[T10]], i32 2 -; AVX-NEXT: [[TMP4:%.*]] = insertelement <8 x i32*> [[TMP3]], i32* [[T14]], i32 3 -; AVX-NEXT: [[TMP5:%.*]] = insertelement <8 x i32*> [[TMP4]], i32* [[T18]], i32 4 -; AVX-NEXT: [[TMP6:%.*]] = insertelement <8 x i32*> [[TMP5]], i32* [[T22]], i32 5 -; AVX-NEXT: [[TMP7:%.*]] = insertelement <8 x i32*> [[TMP6]], i32* [[T26]], i32 6 -; AVX-NEXT: [[TMP8:%.*]] = insertelement <8 x i32*> [[TMP7]], i32* [[T30]], i32 7 -; AVX-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP8]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX-NEXT: [[TMP10:%.*]] = add <8 x i32> [[TMP9]], -; AVX-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0:%.*]] to <8 x i32>* -; AVX-NEXT: store <8 x i32> [[TMP10]], <8 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]] +; AVX-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]] +; AVX-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]] +; AVX-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]] +; AVX-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; AVX-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; AVX-NEXT: [[T8:%.*]] = add i32 [[T7]], 2 +; AVX-NEXT: [[T12:%.*]] = add i32 [[T11]], 3 +; AVX-NEXT: [[T16:%.*]] = add i32 [[T15]], 4 +; AVX-NEXT: [[T20:%.*]] = add i32 [[T19]], 1 +; AVX-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; AVX-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; AVX-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; AVX-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_4( -; AVX2-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11 -; AVX2-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4 -; AVX2-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15 -; AVX2-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18 +; AVX2-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1 +; AVX2-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> poison, i32* [[T1:%.*]], i32 0 +; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer +; AVX2-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> +; AVX2-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5 ; AVX2-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9 +; AVX2-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6 ; AVX2-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 +; AVX2-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX2-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX2-NEXT: [[TMP1:%.*]] = insertelement <8 x i32*> poison, i32* [[T1]], i32 0 -; AVX2-NEXT: [[TMP2:%.*]] = insertelement <8 x i32*> [[TMP1]], i32* [[T6]], i32 1 -; AVX2-NEXT: [[TMP3:%.*]] = insertelement <8 x i32*> [[TMP2]], i32* [[T10]], i32 2 -; AVX2-NEXT: [[TMP4:%.*]] = insertelement <8 x i32*> [[TMP3]], i32* [[T14]], i32 3 -; AVX2-NEXT: [[TMP5:%.*]] = insertelement <8 x i32*> [[TMP4]], i32* [[T18]], i32 4 -; AVX2-NEXT: [[TMP6:%.*]] = insertelement <8 x i32*> [[TMP5]], i32* [[T22]], i32 5 -; AVX2-NEXT: [[TMP7:%.*]] = insertelement <8 x i32*> [[TMP6]], i32* [[T26]], i32 6 -; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i32*> [[TMP7]], i32* [[T30]], i32 7 -; AVX2-NEXT: [[TMP9:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP8]], i32 4, <8 x i1> , <8 x i32> undef), !tbaa [[TBAA0]] -; AVX2-NEXT: [[TMP10:%.*]] = add <8 x i32> [[TMP9]], -; AVX2-NEXT: [[TMP11:%.*]] = bitcast i32* [[T0:%.*]] to <8 x i32>* -; AVX2-NEXT: store <8 x i32> [[TMP10]], <8 x i32>* [[TMP11]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX2-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] +; AVX2-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 +; AVX2-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], +; AVX2-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 +; AVX2-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 +; AVX2-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 +; AVX2-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] +; AVX2-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>* +; AVX2-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX2-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_4( @@ -290,22 +436,22 @@ define void @gather_load_4(i32* noalias nocapture %t0, i32* noalias nocapture re ; AVX512-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6 ; AVX512-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7 ; AVX512-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21 -; AVX512-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), !tbaa [[TBAA0]] -; AVX512-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]] +; AVX512-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> , <4 x i32> undef), [[TBAA0]] +; AVX512-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]] +; AVX512-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]] +; AVX512-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]] ; AVX512-NEXT: [[T4:%.*]] = add i32 [[T3]], 1 ; AVX512-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], ; AVX512-NEXT: [[T24:%.*]] = add i32 [[T23]], 2 ; AVX512-NEXT: [[T28:%.*]] = add i32 [[T27]], 3 ; AVX512-NEXT: [[T32:%.*]] = add i32 [[T31]], 4 -; AVX512-NEXT: store i32 [[T4]], i32* [[T0]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]] ; AVX512-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>* -; AVX512-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T24]], i32* [[T21]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T28]], i32* [[T25]], align 4, !tbaa [[TBAA0]] -; AVX512-NEXT: store i32 [[T32]], i32* [[T29]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]] +; AVX512-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %t5 = getelementptr inbounds i32, i32* %t0, i64 1 @@ -363,21 +509,21 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; SSE-NEXT: [[TMP7:%.*]] = insertelement <4 x float*> [[TMP6]], float* [[TMP3]], i32 1 ; SSE-NEXT: [[TMP8:%.*]] = insertelement <4 x float*> [[TMP7]], float* [[TMP4]], i32 2 ; SSE-NEXT: [[TMP9:%.*]] = insertelement <4 x float*> [[TMP8]], float* [[TMP5]], i32 3 -; SSE-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP9]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP9]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP11:%.*]] = shufflevector <4 x float*> [[TMP6]], <4 x float*> undef, <4 x i32> zeroinitializer ; SSE-NEXT: [[TMP12:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP12]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP12]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP14:%.*]] = fdiv <4 x float> [[TMP10]], [[TMP13]] ; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, float* [[TMP0:%.*]], i64 4 ; SSE-NEXT: [[TMP16:%.*]] = bitcast float* [[TMP0]] to <4 x float>* -; SSE-NEXT: store <4 x float> [[TMP14]], <4 x float>* [[TMP16]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: store <4 x float> [[TMP14]], <4 x float>* [[TMP16]], align 4, [[TBAA0]] ; SSE-NEXT: [[TMP17:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP17]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP17]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP19:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> -; SSE-NEXT: [[TMP20:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP19]], i32 4, <4 x i1> , <4 x float> undef), !tbaa [[TBAA0]] +; SSE-NEXT: [[TMP20:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP19]], i32 4, <4 x i1> , <4 x float> undef), [[TBAA0]] ; SSE-NEXT: [[TMP21:%.*]] = fdiv <4 x float> [[TMP18]], [[TMP20]] ; SSE-NEXT: [[TMP22:%.*]] = bitcast float* [[TMP15]] to <4 x float>* -; SSE-NEXT: store <4 x float> [[TMP21]], <4 x float>* [[TMP22]], align 4, !tbaa [[TBAA0]] +; SSE-NEXT: store <4 x float> [[TMP21]], <4 x float>* [[TMP22]], align 4, [[TBAA0]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @gather_load_div( @@ -396,13 +542,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX-NEXT: ret void ; ; AVX2-LABEL: @gather_load_div( @@ -421,13 +567,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX2-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX2-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX2-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX2-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX2-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX2-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX2-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX2-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX2-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX2-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX2-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX2-NEXT: ret void ; ; AVX512-LABEL: @gather_load_div( @@ -446,13 +592,13 @@ define void @gather_load_div(float* noalias nocapture %0, float* noalias nocaptu ; AVX512-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5 ; AVX512-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6 ; AVX512-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7 -; AVX512-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer ; AVX512-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> -; AVX512-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), !tbaa [[TBAA0]] +; AVX512-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> , <8 x float> undef), [[TBAA0]] ; AVX512-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]] ; AVX512-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>* -; AVX512-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, !tbaa [[TBAA0]] +; AVX512-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]] ; AVX512-NEXT: ret void ; %3 = load float, float* %1, align 4, !tbaa !2 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll index 91e308f..5548a82 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll @@ -118,13 +118,12 @@ define void @tiny_tree_not_fully_vectorizable(double* noalias nocapture %dst, do ; CHECK-NEXT: [[I_015:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[DST_ADDR_014:%.*]] = phi double* [ [[ADD_PTR4:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ] ; CHECK-NEXT: [[SRC_ADDR_013:%.*]] = phi double* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = load double, double* [[SRC_ADDR_013]], align 8 +; CHECK-NEXT: store double [[TMP0]], double* [[DST_ADDR_014]], align 8 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 2 -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double*> poison, double* [[SRC_ADDR_013]], i32 0 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double*> [[TMP0]], double* [[ARRAYIDX2]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[TMP1]], i32 8, <2 x i1> , <2 x double> undef) +; CHECK-NEXT: [[TMP1:%.*]] = load double, double* [[ARRAYIDX2]], align 8 ; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 1 -; CHECK-NEXT: [[TMP3:%.*]] = bitcast double* [[DST_ADDR_014]] to <2 x double>* -; CHECK-NEXT: store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 8 +; CHECK-NEXT: store double [[TMP1]], double* [[ARRAYIDX3]], align 8 ; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 [[I_015]] ; CHECK-NEXT: [[ADD_PTR4]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 [[I_015]] ; CHECK-NEXT: [[INC]] = add i64 [[I_015]], 1 @@ -166,19 +165,20 @@ define void @tiny_tree_not_fully_vectorizable2(float* noalias nocapture %dst, fl ; CHECK-NEXT: [[I_023:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[DST_ADDR_022:%.*]] = phi float* [ [[ADD_PTR8:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ] ; CHECK-NEXT: [[SRC_ADDR_021:%.*]] = phi float* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC_ADDR_021]], align 4 +; CHECK-NEXT: store float [[TMP0]], float* [[DST_ADDR_022]], align 4 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 4 +; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARRAYIDX2]], align 4 ; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 1 +; CHECK-NEXT: store float [[TMP1]], float* [[ARRAYIDX3]], align 4 ; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[ARRAYIDX4]], align 4 ; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 2 +; CHECK-NEXT: store float [[TMP2]], float* [[ARRAYIDX5]], align 4 ; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 3 -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float*> poison, float* [[SRC_ADDR_021]], i32 0 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float*> [[TMP0]], float* [[ARRAYIDX2]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float*> [[TMP1]], float* [[ARRAYIDX4]], i32 2 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float*> [[TMP2]], float* [[ARRAYIDX6]], i32 3 -; CHECK-NEXT: [[TMP4:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP3]], i32 4, <4 x i1> , <4 x float> undef) +; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[ARRAYIDX6]], align 4 ; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 3 -; CHECK-NEXT: [[TMP5:%.*]] = bitcast float* [[DST_ADDR_022]] to <4 x float>* -; CHECK-NEXT: store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4 +; CHECK-NEXT: store float [[TMP3]], float* [[ARRAYIDX7]], align 4 ; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 [[I_023]] ; CHECK-NEXT: [[ADD_PTR8]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 [[I_023]] ; CHECK-NEXT: [[INC]] = add i64 [[I_023]], 1 -- 2.7.4