From c9602e02fc16e8a3fcb6f7b58d8615b04a8fde38 Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Tue, 20 Dec 2022 01:39:31 +0000 Subject: [PATCH] [SVE] Fix incorrect VT usage when lowering fixed length vector divides. Ensure the negation required when lowering negative power-of-two divides uses the scalable vector container type with the fixed length result extracted from it. Fixes: #59647 Differential Revision: https://reviews.llvm.org/D140563 --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 6 ++- .../CodeGen/AArch64/sve-fixed-length-sdiv-pow2.ll | 43 +++++++++++++--------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index e5e314c..e14344e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22954,9 +22954,11 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE( SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), dl, MVT::i32); SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT); - SDValue Res = DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, ContainerVT, Pg, Op1, Op2); + SDValue Res = + DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, ContainerVT, Pg, Op1, Op2); if (Negated) - Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); + Res = DAG.getNode(ISD::SUB, dl, ContainerVT, + DAG.getConstant(0, dl, ContainerVT), Res); return convertFromScalableVector(DAG, VT, Res); } diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-sdiv-pow2.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-sdiv-pow2.ll index 108d41f..d291b33 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-sdiv-pow2.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-sdiv-pow2.ll @@ -10,10 +10,11 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #5 +; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #1 +; CHECK-NEXT: subr z0.b, z0.b, #0 // =0x0 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret - %res = sdiv <8 x i8> %op1, shufflevector (<8 x i8> insertelement (<8 x i8> poison, i8 32, i32 0), <8 x i8> poison, <8 x i32> zeroinitializer) + %res = sdiv <8 x i8> %op1, shufflevector (<8 x i8> insertelement (<8 x i8> poison, i8 -2, i32 0), <8 x i8> poison, <8 x i32> zeroinitializer) ret <8 x i8> %res } @@ -74,11 +75,12 @@ define void @sdiv_v128i8(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] -; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #5 +; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #2 +; CHECK-NEXT: subr z0.b, z0.b, #0 // =0x0 ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <128 x i8>, ptr %a - %res = sdiv <128 x i8> %op1, shufflevector (<128 x i8> insertelement (<128 x i8> poison, i8 32, i32 0), <128 x i8> poison, <128 x i32> zeroinitializer) + %res = sdiv <128 x i8> %op1, shufflevector (<128 x i8> insertelement (<128 x i8> poison, i8 -4, i32 0), <128 x i8> poison, <128 x i32> zeroinitializer) store <128 x i8> %res, ptr %a ret void } @@ -114,10 +116,11 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #5 +; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #3 +; CHECK-NEXT: subr z0.h, z0.h, #0 // =0x0 ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret - %res = sdiv <8 x i16> %op1, shufflevector (<8 x i16> insertelement (<8 x i16> poison, i16 32, i32 0), <8 x i16> poison, <8 x i32> zeroinitializer) + %res = sdiv <8 x i16> %op1, shufflevector (<8 x i16> insertelement (<8 x i16> poison, i16 -8, i32 0), <8 x i16> poison, <8 x i32> zeroinitializer) ret <8 x i16> %res } @@ -166,11 +169,12 @@ define void @sdiv_v64i16(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] -; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #5 +; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #4 +; CHECK-NEXT: subr z0.h, z0.h, #0 // =0x0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <64 x i16>, ptr %a - %res = sdiv <64 x i16> %op1, shufflevector (<64 x i16> insertelement (<64 x i16> poison, i16 32, i32 0), <64 x i16> poison, <64 x i32> zeroinitializer) + %res = sdiv <64 x i16> %op1, shufflevector (<64 x i16> insertelement (<64 x i16> poison, i16 -16, i32 0), <64 x i16> poison, <64 x i32> zeroinitializer) store <64 x i16> %res, ptr %a ret void } @@ -195,9 +199,10 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %op1) vscale_range(2,0) #0 { ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: asrd z0.s, p0/m, z0.s, #5 +; CHECK-NEXT: subr z0.s, z0.s, #0 // =0x0 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret - %res = sdiv <2 x i32> %op1, shufflevector (<2 x i32> insertelement (<2 x i32> poison, i32 32, i32 0), <2 x i32> poison, <2 x i32> zeroinitializer) + %res = sdiv <2 x i32> %op1, shufflevector (<2 x i32> insertelement (<2 x i32> poison, i32 -32, i32 0), <2 x i32> poison, <2 x i32> zeroinitializer) ret <2 x i32> %res } @@ -218,11 +223,12 @@ define void @sdiv_v8i32(ptr %a) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: asrd z0.s, p0/m, z0.s, #5 +; CHECK-NEXT: asrd z0.s, p0/m, z0.s, #6 +; CHECK-NEXT: subr z0.s, z0.s, #0 // =0x0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <8 x i32>, ptr %a - %res = sdiv <8 x i32> %op1, shufflevector (<8 x i32> insertelement (<8 x i32> poison, i32 32, i32 0), <8 x i32> poison, <8 x i32> zeroinitializer) + %res = sdiv <8 x i32> %op1, shufflevector (<8 x i32> insertelement (<8 x i32> poison, i32 -64, i32 0), <8 x i32> poison, <8 x i32> zeroinitializer) store <8 x i32> %res, ptr %a ret void } @@ -286,10 +292,11 @@ define <1 x i64> @sdiv_v1i64(<1 x i64> %op1) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: ptrue p0.d, vl1 -; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #5 +; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #7 +; CHECK-NEXT: subr z0.d, z0.d, #0 // =0x0 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret - %res = sdiv <1 x i64> %op1, shufflevector (<1 x i64> insertelement (<1 x i64> poison, i64 32, i32 0), <1 x i64> poison, <1 x i32> zeroinitializer) + %res = sdiv <1 x i64> %op1, shufflevector (<1 x i64> insertelement (<1 x i64> poison, i64 -128, i32 0), <1 x i64> poison, <1 x i32> zeroinitializer) ret <1 x i64> %res } @@ -311,11 +318,12 @@ define void @sdiv_v4i64(ptr %a) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #5 +; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #8 +; CHECK-NEXT: subr z0.d, z0.d, #0 // =0x0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <4 x i64>, ptr %a - %res = sdiv <4 x i64> %op1, shufflevector (<4 x i64> insertelement (<4 x i64> poison, i64 32, i32 0), <4 x i64> poison, <4 x i32> zeroinitializer) + %res = sdiv <4 x i64> %op1, shufflevector (<4 x i64> insertelement (<4 x i64> poison, i64 -256, i32 0), <4 x i64> poison, <4 x i32> zeroinitializer) store <4 x i64> %res, ptr %a ret void } @@ -365,11 +373,12 @@ define void @sdiv_v32i64(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #5 +; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #9 +; CHECK-NEXT: subr z0.d, z0.d, #0 // =0x0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i64>, ptr %a - %res = sdiv <32 x i64> %op1, shufflevector (<32 x i64> insertelement (<32 x i64> poison, i64 32, i32 0), <32 x i64> poison, <32 x i32> zeroinitializer) + %res = sdiv <32 x i64> %op1, shufflevector (<32 x i64> insertelement (<32 x i64> poison, i64 -512, i32 0), <32 x i64> poison, <32 x i32> zeroinitializer) store <32 x i64> %res, ptr %a ret void } -- 2.7.4