From dfac521da1b90db6832a0d357f67cb819ea8687f Mon Sep 17 00:00:00 2001 From: David Green Date: Thu, 21 Jan 2021 15:30:39 +0000 Subject: [PATCH] [ARM] Fix vector saddsat costs. It turns out the vectorizer calls the getIntrinsicInstrCost functions with a scalar return type and vector VF. This updates the costmodel to handle that, still producing the correct vector costs. A vectorizer test is added to show it vectorizing at the correct factor again. --- llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp | 7 ++- .../LoopVectorize/ARM/mve-saddsatcost.ll | 57 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index a94d351..46c5ba1 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1531,8 +1531,13 @@ int ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, case Intrinsic::usub_sat: { if (!ST->hasMVEIntegerOps()) break; + // Get the Return type, either directly of from ICA.ReturnType and ICA.VF. + Type *VT = ICA.getReturnType(); + if (!VT->isVectorTy() && !ICA.getVectorFactor().isScalar()) + VT = VectorType::get(VT, ICA.getVectorFactor()); + std::pair LT = - TLI->getTypeLegalizationCost(DL, ICA.getReturnType()); + TLI->getTypeLegalizationCost(DL, VT); if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 || LT.second == MVT::v16i8) { // This is a base cost of 1 for the vadd, plus 3 extract shifts if we diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll new file mode 100644 index 0000000..35a3153 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll @@ -0,0 +1,57 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -loop-vectorize -instcombine -simplifycfg < %s -S -o - | FileCheck %s --check-prefix=CHECK + +target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" +target triple = "thumbv8.1m.main-arm-none-eabi" + +define void @arm_offset_q15(i16* nocapture readonly %pSrc, i16 signext %offset, i16* nocapture noalias %pDst, i32 %blockSize) #0 { +; CHECK-LABEL: @arm_offset_q15( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP_NOT6:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0 +; CHECK-NEXT: br i1 [[CMP_NOT6]], label [[WHILE_END:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[BLOCKSIZE]], 7 +; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <8 x i16> poison, i16 [[OFFSET:%.*]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT8]], <8 x i16> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[PSRC:%.*]], i32 [[INDEX]] +; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i16, i16* [[PDST:%.*]], i32 [[INDEX]] +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]]) +; CHECK-NEXT: [[TMP0:%.*]] = bitcast i16* [[NEXT_GEP]] to <8 x i16>* +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* [[TMP0]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison) +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]], <8 x i16> [[BROADCAST_SPLAT9]]) +; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16* [[NEXT_GEP5]] to <8 x i16>* +; CHECK-NEXT: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> [[TMP1]], <8 x i16>* [[TMP2]], i32 2, <8 x i1> [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP3]], label [[WHILE_END]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]] +; CHECK: while.end: +; CHECK-NEXT: ret void +; +entry: + %cmp.not6 = icmp eq i32 %blockSize, 0 + br i1 %cmp.not6, label %while.end, label %while.body + +while.body: ; preds = %entry, %while.body + %blkCnt.09 = phi i32 [ %dec, %while.body ], [ %blockSize, %entry ] + %pSrc.addr.08 = phi i16* [ %incdec.ptr, %while.body ], [ %pSrc, %entry ] + %pDst.addr.07 = phi i16* [ %incdec.ptr3, %while.body ], [ %pDst, %entry ] + %incdec.ptr = getelementptr inbounds i16, i16* %pSrc.addr.08, i32 1 + %0 = load i16, i16* %pSrc.addr.08, align 2 + %1 = tail call i16 @llvm.sadd.sat.i16(i16 %0, i16 %offset) + %incdec.ptr3 = getelementptr inbounds i16, i16* %pDst.addr.07, i32 1 + store i16 %1, i16* %pDst.addr.07, align 2 + %dec = add i32 %blkCnt.09, -1 + %cmp.not = icmp eq i32 %dec, 0 + br i1 %cmp.not, label %while.end, label %while.body + +while.end: ; preds = %while.body, %entry + ret void +} + +declare i16 @llvm.sadd.sat.i16(i16, i16) + +attributes #0 = { "target-features"="+mve" } -- 2.7.4