From 59ff9d3777701ebbe6a59ab2edb8792ef3d2873f Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Mon, 20 Mar 2023 10:21:12 -0700 Subject: [PATCH] [SLP]Fix PR61554: use of missing vectorized value in buildvector nodes. If the buildvector node matches the vector node, it reuse the vector value from this vector node, but its VectorizedValue field is not updated. Need to update this field to avoid misses during the analysis of the reused gather/buildvector nodes. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 15 +++++++++++++++ .../X86/reused-buildvector-matching-vectorized-node.ll | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 9da2fd4..ba8e045 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9352,6 +9352,21 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) { V = FinalShuffle(V, UniformMask); } } + // Need to update the operand gather node, if actually the operand is not a + // vectorized node, but the buildvector/gather node, which matches one of + // the vectorized nodes. + if (find_if(VE->UserTreeIndices, [&](const EdgeInfo &EI) { + return EI.UserTE == E && EI.EdgeIdx == NodeIdx; + }) == VE->UserTreeIndices.end()) { + auto *It = find_if( + VectorizableTree, [&](const std::unique_ptr &TE) { + return TE->State == TreeEntry::NeedToGather && + TE->UserTreeIndices.front().UserTE == E && + TE->UserTreeIndices.front().EdgeIdx == NodeIdx; + }); + assert(It != VectorizableTree.end() && "Expected gather node operand."); + (*It)->VectorizedValue = V; + } return V; } } diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-buildvector-matching-vectorized-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-buildvector-matching-vectorized-node.ll index d250fcf..2b425ee 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-buildvector-matching-vectorized-node.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-buildvector-matching-vectorized-node.ll @@ -10,7 +10,7 @@ define void @blam(ptr %arg, double %load2, i1 %fcmp3) { ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i1> poison, i1 [[FCMP3]], i32 0 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[TMP0]] -; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr poison, align 16 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP0]], double [[LOAD2]], i32 0 ; CHECK-NEXT: [[TMP5:%.*]] = fcmp olt <2 x double> [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> zeroinitializer, <2 x double> [[TMP0]] ; CHECK-NEXT: [[TMP7:%.*]] = fcmp olt <2 x double> [[TMP3]], zeroinitializer -- 2.7.4