[X86] Give priority to EVEX FMA instructions over FMA4 instructions.

No existing processor has both so it doesn't really matter what we do here. But we were previously just relying on pattern order which gave FMA4 priority.

llvm-svn: 317775

diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 84b44ac..e106084 100644 (file)
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -6002,12 +6002,12 @@ let Constraints = "$src1 = $dst", hasSideEffects = 0 in {
          AVX512FMA3Base, EVEX_B, EVEX_RC;
 
   let isCodeGenOnly = 1, isCommutable = 1 in {
-    def r     : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst),
+    def r     : AVX512FMA3S<opc, MRMSrcReg, (outs _.FRC:$dst),
                      (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
                      !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                      !if(MaskOnlyReg, [], [RHS_r])>;
-    def m     : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst),
+    def m     : AVX512FMA3S<opc, MRMSrcMem, (outs _.FRC:$dst),
                     (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
                     !strconcat(OpcodeStr,
                                "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),

diff --git a/llvm/lib/Target/X86/X86InstrFMA.td b/llvm/lib/Target/X86/X86InstrFMA.td
index 15466c2..b1f5d44 100644 (file)
--- a/llvm/lib/Target/X86/X86InstrFMA.td
+++ b/llvm/lib/Target/X86/X86InstrFMA.td
@@ -168,59 +168,59 @@ let ExeDomain = SSEPackedDouble in {
 multiclass fma3s_rm_213<bits<8> opc, string OpcodeStr,
                         X86MemOperand x86memop, RegisterClass RC,
                         SDPatternOperator OpNode> {
-  def r : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, RC:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               [(set RC:$dst, (OpNode RC:$src2, RC:$src1, RC:$src3))]>;
+  def r : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, RC:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                [(set RC:$dst, (OpNode RC:$src2, RC:$src1, RC:$src3))]>;
 
   let mayLoad = 1 in
-  def m : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, x86memop:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               [(set RC:$dst,
-                 (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>;
+  def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, x86memop:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                [(set RC:$dst,
+                  (OpNode RC:$src2, RC:$src1, (load addr:$src3)))]>;
 }
 
 multiclass fma3s_rm_231<bits<8> opc, string OpcodeStr,
                         X86MemOperand x86memop, RegisterClass RC,
                         SDPatternOperator OpNode> {
   let hasSideEffects = 0 in
-  def r : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, RC:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               []>;
+  def r : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, RC:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                []>;
 
   let mayLoad = 1 in
-  def m : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, x86memop:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               [(set RC:$dst,
-                 (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>;
+  def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, x86memop:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                [(set RC:$dst,
+                  (OpNode RC:$src2, (load addr:$src3), RC:$src1))]>;
 }
 
 multiclass fma3s_rm_132<bits<8> opc, string OpcodeStr,
                         X86MemOperand x86memop, RegisterClass RC,
                         SDPatternOperator OpNode> {
   let hasSideEffects = 0 in
-  def r : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, RC:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               []>;
+  def r : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, RC:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                []>;
 
   // Pattern is 312 order so that the load is in a different place from the
   // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
   let mayLoad = 1 in
-  def m : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, x86memop:$src3),
-               !strconcat(OpcodeStr,
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-               [(set RC:$dst,
-                 (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>;
+  def m : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
+                (ins RC:$src1, RC:$src2, x86memop:$src3),
+                !strconcat(OpcodeStr,
+                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                [(set RC:$dst,
+                  (OpNode (load addr:$src3), RC:$src1, RC:$src2))]>;
 }
 
 let Constraints = "$src1 = $dst", isCommutable = 1, hasSideEffects = 0 in
@@ -228,14 +228,12 @@ multiclass fma3s_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
                        string OpStr, string PackTy, string Suff,
                        SDNode OpNode, RegisterClass RC,
                        X86MemOperand x86memop> {
-  let Predicates = [HasFMA, NoAVX512] in {
-    defm NAME#213#Suff : fma3s_rm_213<opc213, !strconcat(OpStr, "213", PackTy),
-                                      x86memop, RC, OpNode>;
-    defm NAME#231#Suff : fma3s_rm_231<opc231, !strconcat(OpStr, "231", PackTy),
-                                      x86memop, RC, OpNode>;
-    defm NAME#132#Suff : fma3s_rm_132<opc132, !strconcat(OpStr, "132", PackTy),
-                                      x86memop, RC, OpNode>;
-  }
+  defm NAME#213#Suff : fma3s_rm_213<opc213, !strconcat(OpStr, "213", PackTy),
+                                    x86memop, RC, OpNode>;
+  defm NAME#231#Suff : fma3s_rm_231<opc231, !strconcat(OpStr, "231", PackTy),
+                                    x86memop, RC, OpNode>;
+  defm NAME#132#Suff : fma3s_rm_132<opc132, !strconcat(OpStr, "132", PackTy),
+                                    x86memop, RC, OpNode>;
 }
 
 // These FMA*_Int instructions are defined specially for being used when
@@ -255,18 +253,18 @@ let Constraints = "$src1 = $dst", isCommutable = 1, isCodeGenOnly = 1,
     hasSideEffects = 0 in
 multiclass fma3s_rm_int<bits<8> opc, string OpcodeStr,
                         Operand memopr, RegisterClass RC> {
-  def r_Int : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr,
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   []>;
+  def r_Int : FMA3S<opc, MRMSrcReg, (outs RC:$dst),
+                    (ins RC:$src1, RC:$src2, RC:$src3),
+                    !strconcat(OpcodeStr,
+                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                    []>;
 
   let mayLoad = 1 in
-  def m_Int : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, memopr:$src3),
-                   !strconcat(OpcodeStr,
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   []>;
+  def m_Int : FMA3S<opc, MRMSrcMem, (outs RC:$dst),
+                    (ins RC:$src1, RC:$src2, memopr:$src3),
+                    !strconcat(OpcodeStr,
+                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                    []>;
 }
 
 // The FMA 213 form is created for lowering of scalar FMA intrinscis
@@ -357,19 +355,19 @@ multiclass fma4s<bits<8> opc, string OpcodeStr, RegisterClass RC,
                  X86MemOperand x86memop, ValueType OpVT, SDNode OpNode,
                  PatFrag mem_frag> {
   let isCommutable = 1 in
-  def rr : FMA4<opc, MRMSrcRegOp4, (outs RC:$dst),
+  def rr : FMA4S<opc, MRMSrcRegOp4, (outs RC:$dst),
            (ins RC:$src1, RC:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst,
              (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, VEX_W, VEX_LIG;
-  def rm : FMA4<opc, MRMSrcMemOp4, (outs RC:$dst),
+  def rm : FMA4S<opc, MRMSrcMemOp4, (outs RC:$dst),
            (ins RC:$src1, RC:$src2, x86memop:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set RC:$dst, (OpNode RC:$src1, RC:$src2,
                            (mem_frag addr:$src3)))]>, VEX_W, VEX_LIG;
-  def mr : FMA4<opc, MRMSrcMem, (outs RC:$dst),
+  def mr : FMA4S<opc, MRMSrcMem, (outs RC:$dst),
            (ins RC:$src1, x86memop:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
@@ -377,7 +375,7 @@ multiclass fma4s<bits<8> opc, string OpcodeStr, RegisterClass RC,
              (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>, VEX_LIG;
 // For disassembler
 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
-  def rr_REV : FMA4<opc, MRMSrcReg, (outs RC:$dst),
+  def rr_REV : FMA4S<opc, MRMSrcReg, (outs RC:$dst),
                (ins RC:$src1, RC:$src2, RC:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
@@ -387,20 +385,20 @@ let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in
 multiclass fma4s_int<bits<8> opc, string OpcodeStr, Operand memop,
                      ValueType VT, ComplexPattern mem_cpat, SDNode OpNode> {
 let isCodeGenOnly = 1 in {
-  def rr_Int : FMA4<opc, MRMSrcRegOp4, (outs VR128:$dst),
+  def rr_Int : FMA4S<opc, MRMSrcRegOp4, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst,
                  (VT (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>, VEX_W,
                VEX_LIG;
-  def rm_Int : FMA4<opc, MRMSrcMemOp4, (outs VR128:$dst),
+  def rm_Int : FMA4S<opc, MRMSrcMemOp4, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, memop:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                [(set VR128:$dst, (VT (OpNode VR128:$src1, VR128:$src2,
                                   mem_cpat:$src3)))]>, VEX_W, VEX_LIG;
-  def mr_Int : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
+  def mr_Int : FMA4S<opc, MRMSrcMem, (outs VR128:$dst),
                (ins VR128:$src1, memop:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
@@ -408,7 +406,7 @@ let isCodeGenOnly = 1 in {
                  (VT (OpNode VR128:$src1, mem_cpat:$src2, VR128:$src3)))]>,
                VEX_LIG;
 let hasSideEffects = 0 in
-  def rr_Int_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
+  def rr_Int_REV : FMA4S<opc, MRMSrcReg, (outs VR128:$dst),
                (ins VR128:$src1, VR128:$src2, VR128:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),

diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td
index 2653e8c..5604aef 100644 (file)
--- a/llvm/lib/Target/X86/X86InstrFormats.td
+++ b/llvm/lib/Target/X86/X86InstrFormats.td
@@ -827,7 +827,7 @@ class AVX512PIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
 class AVX512PI<bits<8> o, Format F, dag outs, dag ins, string asm,
               list<dag> pattern, Domain d, InstrItinClass itin = NoItinerary>
       : I<o, F, outs, ins, asm, pattern, itin, d>, Requires<[HasAVX512]>;
-class AVX512FMA3<bits<8> o, Format F, dag outs, dag ins, string asm,
+class AVX512FMA3S<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag>pattern, InstrItinClass itin = NoItinerary>
       : I<o, F, outs, ins, asm, pattern, itin>, T8PD,
         EVEX_4V, Requires<[HasAVX512]>;
@@ -867,12 +867,20 @@ class FMA3<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag>pattern, InstrItinClass itin = NoItinerary>
       : I<o, F, outs, ins, asm, pattern, itin>, T8PD,
         VEX_4V, FMASC, Requires<[HasFMA, NoVLX]>;
+class FMA3S<bits<8> o, Format F, dag outs, dag ins, string asm,
+            list<dag>pattern, InstrItinClass itin = NoItinerary>
+      : I<o, F, outs, ins, asm, pattern, itin>, T8PD,
+        VEX_4V, FMASC, Requires<[HasFMA, NoAVX512]>;
 
 // FMA4 Instruction Templates
 class FMA4<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag>pattern, InstrItinClass itin = NoItinerary>
       : Ii8Reg<o, F, outs, ins, asm, pattern, itin>, TAPD,
-        VEX_4V, FMASC, Requires<[HasFMA4]>;
+        VEX_4V, FMASC, Requires<[HasFMA4, NoVLX]>;
+class FMA4S<bits<8> o, Format F, dag outs, dag ins, string asm,
+            list<dag>pattern, InstrItinClass itin = NoItinerary>
+      : Ii8Reg<o, F, outs, ins, asm, pattern, itin>, TAPD,
+        VEX_4V, FMASC, Requires<[HasFMA4, NoAVX512]>;
 
 // XOP 2, 3 and 4 Operand Instruction Template
 class IXOP<bits<8> o, Format F, dag outs, dag ins, string asm,