From 15b5d1a5bf1f179835247d34e4febad1e2ee3121 Mon Sep 17 00:00:00 2001 From: David Green Date: Mon, 3 May 2021 00:04:21 +0100 Subject: [PATCH] [ARM] Transfer memory operands for VLDn We create MMO's for the VLDn/VSTn intrinsics in ARMTargetLowering:: getTgtMemIntrinsic, but they do not currently make it ll the way through ISel. This changes that in the various places it needs changing, making sure that the MMO is propagate through to the final instruction. This can help in scheduling, not treating the VLD2/VST2 as a scheduling barrier. Differential Revision: https://reviews.llvm.org/D101096 --- llvm/include/llvm/IR/IntrinsicsARM.td | 14 ++++++++----- llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp | 2 ++ llvm/lib/Target/ARM/ARMInstrMVE.td | 4 ++-- llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll | 4 ++-- llvm/test/CodeGen/Thumb2/mve-vld2.ll | 28 +++++++++++++------------- llvm/test/CodeGen/Thumb2/mve-vld4.ll | 22 ++++++++++---------- llvm/test/CodeGen/Thumb2/mve-vst2.ll | 12 +++++------ llvm/test/CodeGen/Thumb2/mve-vst4.ll | 4 ++-- 8 files changed, 48 insertions(+), 42 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td index 0eb27cc..2a2c9c7 100644 --- a/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/llvm/include/llvm/IR/IntrinsicsARM.td @@ -1132,11 +1132,15 @@ defm int_arm_mve_vcmlaq : MVEPredicated< [llvm_i32_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], llvm_anyvector_ty>; -def int_arm_mve_vld2q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly]>; -def int_arm_mve_vld4q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly]>; - -def int_arm_mve_vst2q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], [IntrWriteMem, IntrArgMemOnly]>; -def int_arm_mve_vst4q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty], [IntrWriteMem, IntrArgMemOnly]>; +def int_arm_mve_vld2q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_anyptr_ty], + [IntrReadMem, IntrArgMemOnly]>; +def int_arm_mve_vld4q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty], + [IntrReadMem, IntrArgMemOnly]>; + +def int_arm_mve_vst2q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], + [IntrWriteMem, IntrArgMemOnly], "", [SDNPMemOperand]>; +def int_arm_mve_vst4q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty], + [IntrWriteMem, IntrArgMemOnly], "", [SDNPMemOperand]>; // MVE vector absolute difference and accumulate across vector // The first operand is an 'unsigned' flag. The remaining operands are: diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp index e13a672..6c247fe 100644 --- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -2764,6 +2764,7 @@ void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs, CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops); Data = SDValue(LoadInst, 0); Chain = SDValue(LoadInst, 1); + transferMemOperands(N, LoadInst); } // The last may need a writeback on it if (HasWriteback) @@ -2771,6 +2772,7 @@ void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs, SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain}; auto LoadInst = CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops); + transferMemOperands(N, LoadInst); unsigned i; for (i = 0; i < NumVecs; i++) diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td index 7fc5029..0356f42 100644 --- a/llvm/lib/Target/ARM/ARMInstrMVE.td +++ b/llvm/lib/Target/ARM/ARMInstrMVE.td @@ -6027,8 +6027,8 @@ def SDTARMVST2 : SDTypeProfile<1, 5, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisV def SDTARMVST4 : SDTypeProfile<1, 7, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>, SDTCisSameAs<3, 4>, SDTCisSameAs<3, 5>, SDTCisSameAs<3, 6>, SDTCisVT<7, i32>]>; -def MVEVST2UPD : SDNode<"ARMISD::VST2_UPD", SDTARMVST2, [SDNPHasChain]>; -def MVEVST4UPD : SDNode<"ARMISD::VST4_UPD", SDTARMVST4, [SDNPHasChain]>; +def MVEVST2UPD : SDNode<"ARMISD::VST2_UPD", SDTARMVST2, [SDNPHasChain, SDNPMemOperand]>; +def MVEVST4UPD : SDNode<"ARMISD::VST4_UPD", SDTARMVST4, [SDNPHasChain, SDNPMemOperand]>; multiclass MVE_vst24_patterns { foreach stage = [0,1] in diff --git a/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll b/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll index 8449b4a..34a599b 100644 --- a/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll +++ b/llvm/test/CodeGen/Thumb2/mve-multivec-spill.ll @@ -19,16 +19,16 @@ define arm_aapcs_vfpcc void @spill_multivector(<4 x i32>* %p) { ; CHECK-NEXT: vld21.32 {q0, q1}, [r5]! ; CHECK-NEXT: adds r0, #64 ; CHECK-NEXT: vstmia lr, {d0, d1, d2, d3} @ 32-byte Spill -; CHECK-NEXT: add.w lr, sp, #32 ; CHECK-NEXT: vld20.32 {q0, q1}, [r0] +; CHECK-NEXT: add.w lr, sp, #32 ; CHECK-NEXT: vld21.32 {q0, q1}, [r0] ; CHECK-NEXT: add.w r0, r4, #128 ; CHECK-NEXT: vstmia lr, {d0, d1, d2, d3} @ 32-byte Spill ; CHECK-NEXT: vld20.32 {q0, q1}, [r0] ; CHECK-NEXT: vld21.32 {q0, q1}, [r0] ; CHECK-NEXT: add.w r0, r4, #192 -; CHECK-NEXT: vstmia sp, {d0, d1, d2, d3} @ 32-byte Spill ; CHECK-NEXT: vld20.32 {q6, q7}, [r0] +; CHECK-NEXT: vstmia sp, {d0, d1, d2, d3} @ 32-byte Spill ; CHECK-NEXT: vld21.32 {q6, q7}, [r0] ; CHECK-NEXT: add.w r0, r4, #256 ; CHECK-NEXT: vld20.32 {q4, q5}, [r0] diff --git a/llvm/test/CodeGen/Thumb2/mve-vld2.ll b/llvm/test/CodeGen/Thumb2/mve-vld2.ll index 8b139f8..5e5328b 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vld2.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vld2.ll @@ -71,22 +71,22 @@ define void @vld2_v16i32(<32 x i32> *%src, <16 x i32> *%dst) { ; CHECK-NEXT: vld20.32 {q0, q1}, [r0] ; CHECK-NEXT: add.w r2, r0, #96 ; CHECK-NEXT: add.w r3, r0, #64 +; CHECK-NEXT: vld20.32 {q3, q4}, [r2] ; CHECK-NEXT: vld21.32 {q0, q1}, [r0]! +; CHECK-NEXT: vld21.32 {q3, q4}, [r2] +; CHECK-NEXT: vld20.32 {q5, q6}, [r0] ; CHECK-NEXT: vadd.i32 q0, q0, q1 ; CHECK-NEXT: vld20.32 {q1, q2}, [r3] -; CHECK-NEXT: vld20.32 {q3, q4}, [r2] -; CHECK-NEXT: vld20.32 {q5, q6}, [r0] +; CHECK-NEXT: vadd.i32 q3, q3, q4 ; CHECK-NEXT: vld21.32 {q5, q6}, [r0] ; CHECK-NEXT: vld21.32 {q1, q2}, [r3] -; CHECK-NEXT: vld21.32 {q3, q4}, [r2] +; CHECK-NEXT: vstrw.32 q3, [r1, #48] ; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q1_q2 -; CHECK-NEXT: vstrw.32 q0, [r1] ; CHECK-NEXT: vadd.i32 q5, q5, q6 +; CHECK-NEXT: vstrw.32 q0, [r1] ; CHECK-NEXT: vadd.i32 q1, q1, q2 -; CHECK-NEXT: vadd.i32 q3, q3, q4 -; CHECK-NEXT: vstrw.32 q1, [r1, #32] -; CHECK-NEXT: vstrw.32 q3, [r1, #48] ; CHECK-NEXT: vstrw.32 q5, [r1, #16] +; CHECK-NEXT: vstrw.32 q1, [r1, #32] ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: bx lr entry: @@ -473,22 +473,22 @@ define void @vld2_v16f32(<32 x float> *%src, <16 x float> *%dst) { ; CHECK-NEXT: vld20.32 {q0, q1}, [r0] ; CHECK-NEXT: add.w r2, r0, #96 ; CHECK-NEXT: add.w r3, r0, #64 +; CHECK-NEXT: vld20.32 {q3, q4}, [r2] ; CHECK-NEXT: vld21.32 {q0, q1}, [r0]! +; CHECK-NEXT: vld21.32 {q3, q4}, [r2] +; CHECK-NEXT: vld20.32 {q5, q6}, [r0] ; CHECK-NEXT: vadd.f32 q0, q0, q1 ; CHECK-NEXT: vld20.32 {q1, q2}, [r3] -; CHECK-NEXT: vld20.32 {q3, q4}, [r2] -; CHECK-NEXT: vld20.32 {q5, q6}, [r0] +; CHECK-NEXT: vadd.f32 q3, q3, q4 ; CHECK-NEXT: vld21.32 {q5, q6}, [r0] ; CHECK-NEXT: vld21.32 {q1, q2}, [r3] -; CHECK-NEXT: vld21.32 {q3, q4}, [r2] +; CHECK-NEXT: vstrw.32 q3, [r1, #48] ; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q1_q2 -; CHECK-NEXT: vstrw.32 q0, [r1] ; CHECK-NEXT: vadd.f32 q5, q5, q6 +; CHECK-NEXT: vstrw.32 q0, [r1] ; CHECK-NEXT: vadd.f32 q1, q1, q2 -; CHECK-NEXT: vadd.f32 q3, q3, q4 -; CHECK-NEXT: vstrw.32 q1, [r1, #32] -; CHECK-NEXT: vstrw.32 q3, [r1, #48] ; CHECK-NEXT: vstrw.32 q5, [r1, #16] +; CHECK-NEXT: vstrw.32 q1, [r1, #32] ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: bx lr entry: diff --git a/llvm/test/CodeGen/Thumb2/mve-vld4.ll b/llvm/test/CodeGen/Thumb2/mve-vld4.ll index 0d574f9..2e37791 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vld4.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vld4.ll @@ -128,19 +128,19 @@ define void @vld4_v16i32(<64 x i32> *%src, <16 x i32> *%dst) { ; CHECK-NEXT: vadd.i32 q4, q2, q3 ; CHECK-NEXT: vadd.i32 q0, q0, q1 ; CHECK-NEXT: vstrw.32 q4, [sp, #112] @ 16-byte Spill -; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill ; CHECK-NEXT: vld40.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill +; CHECK-NEXT: vldrw.u32 q6, [sp, #112] @ 16-byte Reload ; CHECK-NEXT: vld41.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vldrw.u32 q5, [sp, #96] @ 16-byte Reload ; CHECK-NEXT: vld42.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vadd.i32 q6, q5, q6 +; CHECK-NEXT: vstrw.32 q6, [sp, #112] @ 16-byte Spill ; CHECK-NEXT: vld43.32 {q1, q2, q3, q4}, [r3] -; CHECK-NEXT: vldrw.u32 q6, [sp, #112] @ 16-byte Reload -; CHECK-NEXT: vldrw.u32 q5, [sp, #96] @ 16-byte Reload ; CHECK-NEXT: vstrw.32 q4, [sp, #80] @ 16-byte Spill ; CHECK-NEXT: vmov q0, q1 -; CHECK-NEXT: vadd.i32 q6, q5, q6 ; CHECK-NEXT: vldrw.u32 q5, [sp, #80] @ 16-byte Reload ; CHECK-NEXT: vadd.i32 q0, q0, q2 -; CHECK-NEXT: vstrw.32 q6, [sp, #112] @ 16-byte Spill ; CHECK-NEXT: vadd.i32 q1, q3, q5 ; CHECK-NEXT: vadd.i32 q0, q0, q1 ; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill @@ -927,19 +927,19 @@ define void @vld4_v16f32(<64 x float> *%src, <16 x float> *%dst) { ; CHECK-NEXT: vadd.f32 q4, q2, q3 ; CHECK-NEXT: vadd.f32 q0, q0, q1 ; CHECK-NEXT: vstrw.32 q4, [sp, #112] @ 16-byte Spill -; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill ; CHECK-NEXT: vld40.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill +; CHECK-NEXT: vldrw.u32 q6, [sp, #112] @ 16-byte Reload ; CHECK-NEXT: vld41.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vldrw.u32 q5, [sp, #96] @ 16-byte Reload ; CHECK-NEXT: vld42.32 {q1, q2, q3, q4}, [r3] +; CHECK-NEXT: vadd.f32 q6, q5, q6 +; CHECK-NEXT: vstrw.32 q6, [sp, #112] @ 16-byte Spill ; CHECK-NEXT: vld43.32 {q1, q2, q3, q4}, [r3] -; CHECK-NEXT: vldrw.u32 q6, [sp, #112] @ 16-byte Reload -; CHECK-NEXT: vldrw.u32 q5, [sp, #96] @ 16-byte Reload ; CHECK-NEXT: vstrw.32 q4, [sp, #80] @ 16-byte Spill ; CHECK-NEXT: vmov q0, q1 -; CHECK-NEXT: vadd.f32 q6, q5, q6 ; CHECK-NEXT: vldrw.u32 q5, [sp, #80] @ 16-byte Reload ; CHECK-NEXT: vadd.f32 q0, q0, q2 -; CHECK-NEXT: vstrw.32 q6, [sp, #112] @ 16-byte Spill ; CHECK-NEXT: vadd.f32 q1, q3, q5 ; CHECK-NEXT: vadd.f32 q0, q0, q1 ; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill @@ -1153,8 +1153,8 @@ define void @vld4_v16f16(<64 x half> *%src, <16 x half> *%dst) { ; CHECK-NEXT: vld41.16 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vld42.16 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vld43.16 {q0, q1, q2, q3}, [r0]! -; CHECK-NEXT: vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill ; CHECK-NEXT: vld40.16 {q4, q5, q6, q7}, [r0] +; CHECK-NEXT: vstmia sp, {d0, d1, d2, d3, d4, d5, d6, d7} @ 64-byte Spill ; CHECK-NEXT: vld41.16 {q4, q5, q6, q7}, [r0] ; CHECK-NEXT: vld42.16 {q4, q5, q6, q7}, [r0] ; CHECK-NEXT: vld43.16 {q4, q5, q6, q7}, [r0] diff --git a/llvm/test/CodeGen/Thumb2/mve-vst2.ll b/llvm/test/CodeGen/Thumb2/mve-vst2.ll index 8529787..67a6063 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vst2.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vst2.ll @@ -75,15 +75,15 @@ define void @vst2_v16i32(<16 x i32> *%src, <32 x i32> *%dst) { ; CHECK-NEXT: vldrw.u32 q0, [r0, #48] ; CHECK-NEXT: vldrw.u32 q2, [r0, #32] ; CHECK-NEXT: vldrw.u32 q4, [r0, #16] -; CHECK-NEXT: vst20.32 {q6, q7}, [r1] ; CHECK-NEXT: add.w r0, r1, #96 ; CHECK-NEXT: add.w r2, r1, #64 +; CHECK-NEXT: vst20.32 {q6, q7}, [r1] ; CHECK-NEXT: vst21.32 {q6, q7}, [r1]! ; CHECK-NEXT: vst20.32 {q4, q5}, [r1] -; CHECK-NEXT: vst21.32 {q4, q5}, [r1] ; CHECK-NEXT: vst20.32 {q2, q3}, [r2] -; CHECK-NEXT: vst21.32 {q2, q3}, [r2] ; CHECK-NEXT: vst20.32 {q0, q1}, [r0] +; CHECK-NEXT: vst21.32 {q4, q5}, [r1] +; CHECK-NEXT: vst21.32 {q2, q3}, [r2] ; CHECK-NEXT: vst21.32 {q0, q1}, [r0] ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: bx lr @@ -460,15 +460,15 @@ define void @vst2_v16f32(<16 x float> *%src, <32 x float> *%dst) { ; CHECK-NEXT: vldrw.u32 q0, [r0, #48] ; CHECK-NEXT: vldrw.u32 q2, [r0, #32] ; CHECK-NEXT: vldrw.u32 q4, [r0, #16] -; CHECK-NEXT: vst20.32 {q6, q7}, [r1] ; CHECK-NEXT: add.w r0, r1, #96 ; CHECK-NEXT: add.w r2, r1, #64 +; CHECK-NEXT: vst20.32 {q6, q7}, [r1] ; CHECK-NEXT: vst21.32 {q6, q7}, [r1]! ; CHECK-NEXT: vst20.32 {q4, q5}, [r1] -; CHECK-NEXT: vst21.32 {q4, q5}, [r1] ; CHECK-NEXT: vst20.32 {q2, q3}, [r2] -; CHECK-NEXT: vst21.32 {q2, q3}, [r2] ; CHECK-NEXT: vst20.32 {q0, q1}, [r0] +; CHECK-NEXT: vst21.32 {q4, q5}, [r1] +; CHECK-NEXT: vst21.32 {q2, q3}, [r2] ; CHECK-NEXT: vst21.32 {q0, q1}, [r0] ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: bx lr diff --git a/llvm/test/CodeGen/Thumb2/mve-vst4.ll b/llvm/test/CodeGen/Thumb2/mve-vst4.ll index eb6af2c..25abf74 100644 --- a/llvm/test/CodeGen/Thumb2/mve-vst4.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vst4.ll @@ -175,10 +175,10 @@ define void @vst4_v16i32(<16 x i32> *%src, <64 x i32> *%dst) { ; CHECK-NEXT: vst40.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst41.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst42.32 {q4, q5, q6, q7}, [r1] -; CHECK-NEXT: vst43.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst40.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vst41.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vst42.32 {q0, q1, q2, q3}, [r0] +; CHECK-NEXT: vst43.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst43.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: add sp, #216 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} @@ -945,10 +945,10 @@ define void @vst4_v16f32(<16 x float> *%src, <64 x float> *%dst) { ; CHECK-NEXT: vst40.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst41.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst42.32 {q4, q5, q6, q7}, [r1] -; CHECK-NEXT: vst43.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst40.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vst41.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: vst42.32 {q0, q1, q2, q3}, [r0] +; CHECK-NEXT: vst43.32 {q4, q5, q6, q7}, [r1] ; CHECK-NEXT: vst43.32 {q0, q1, q2, q3}, [r0] ; CHECK-NEXT: add sp, #216 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} -- 2.7.4