From 8a47e6252ad43a2eb3238f9b36394571ba13f4a9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 5 Jan 2021 14:56:05 +0000 Subject: [PATCH] [VPlan] Re-add interleave group members to plan. Creating in-loop reductions relies on IR references to map IR values to VPValues after interleave group creation. Make sure we re-add the updated member to the plan, so the look-ups still work as expected This fixes a crash reported after D90562. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 1 + .../Transforms/LoopVectorize/ARM/mve-reductions.ll | 81 ++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 7f89fd9a..2bd35cc 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8520,6 +8520,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( if (!Member->getType()->isVoidTy()) { VPValue *OriginalV = Plan->getVPValue(Member); Plan->removeVPValueFor(Member); + Plan->addVPValue(Member, VPIG->getVPValue(J)); OriginalV->replaceAllUsesWith(VPIG->getVPValue(J)); J++; } diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll index fad08b9..0628aa8 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll @@ -950,4 +950,85 @@ for.cond.cleanup: ; preds = %for.body, %entry ret i8 %r.0.lcssa } +; Make sure interleave group members feeding in-loop reductions can be handled. +define i32 @reduction_interleave_group(i32 %n, i32* %arr) #0 { +; CHECK-LABEL: @reduction_interleave_group( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[GUARD:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECK-NEXT: br i1 [[GUARD]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK: for.body.preheader: +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 6 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -4 +; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 1 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = or i32 [[OFFSET_IDX]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i32 -1 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 [[TMP3]] +; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <8 x i32>* +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP6]], align 4 +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> +; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]]) +; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]]) +; CHECK-NEXT: [[TMP10]] = add i32 [[TMP9]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP13:!llvm.loop !.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ADD:%.*]] = or i32 [[IV]], 1 +; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[ADD]] +; CHECK-NEXT: [[L_0:%.*]] = load i32, i32* [[GEP_0]], align 4 +; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[IV]] +; CHECK-NEXT: [[L_1:%.*]] = load i32, i32* [[GEP_1]], align 4 +; CHECK-NEXT: [[RED_1:%.*]] = add i32 [[L_0]], [[RED_PHI]] +; CHECK-NEXT: [[RED_2]] = add i32 [[RED_1]], [[L_1]] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 2 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], [[LOOP14:!llvm.loop !.*]] +; CHECK: exit: +; CHECK-NEXT: [[RET_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RED_2]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[RET_LCSSA]] +; +entry: + %guard = icmp sgt i32 %n, 0 + br i1 %guard , label %for.body, label %exit + +for.body: ; preds = %for.body.preheader, %for.body + %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ] + %red.phi = phi i32 [ %red.2, %for.body ], [ 0, %entry ] + %add = or i32 %iv, 1 + %gep.0 = getelementptr inbounds i32, i32* %arr, i32 %add + %l.0 = load i32, i32* %gep.0, align 4 + %gep.1 = getelementptr inbounds i32, i32* %arr, i32 %iv + %l.1 = load i32, i32* %gep.1, align 4 + %red.1 = add i32 %l.0, %red.phi + %red.2 = add i32 %red.1, %l.1 + %iv.next = add nuw nsw i32 %iv, 2 + %cmp = icmp slt i32 %iv.next, %n + br i1 %cmp, label %for.body, label %exit + +exit: + %ret.lcssa = phi i32 [ 0, %entry ], [ %red.2, %for.body ] + ret i32 %ret.lcssa +} + attributes #0 = { "target-features"="+mve" } -- 2.7.4