From c9d7484aa36e974a265b88dfbaa75cf8ea7cd5bc Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Sat, 11 May 2019 16:00:28 +0000 Subject: [PATCH] [X86] Add CMOV_FR32X/CMOV_FR64X pseudo instructions. Use them in fast isel to fix a machine verifier error after adding test cases. Fast isel picks the FR32X/FR64X register classes when lowering pseudo select, but it didn't have the right opcode to go with it. llvm-svn: 360524 --- llvm/lib/Target/X86/X86FastISel.cpp | 6 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 2 + llvm/lib/Target/X86/X86InstrCompiler.td | 10 +- .../CodeGen/X86/fast-isel-select-pseudo-cmov.ll | 196 +++++++++++++++++++++ 4 files changed, 210 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index e9202a6..c397d70 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -2281,8 +2281,10 @@ bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) { case MVT::i8: Opc = X86::CMOV_GR8; break; case MVT::i16: Opc = X86::CMOV_GR16; break; case MVT::i32: Opc = X86::CMOV_GR32; break; - case MVT::f32: Opc = X86::CMOV_FR32; break; - case MVT::f64: Opc = X86::CMOV_FR64; break; + case MVT::f32: Opc = Subtarget->hasAVX512() ? X86::CMOV_FR32X + : X86::CMOV_FR32; break; + case MVT::f64: Opc = Subtarget->hasAVX512() ? X86::CMOV_FR64X + : X86::CMOV_FR64; break; } const Value *Cond = I->getOperand(0); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index f7f3d38..22f05a5 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -30477,7 +30477,9 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, case X86::TLSCall_64: return EmitLoweredTLSCall(MI, BB); case X86::CMOV_FR32: + case X86::CMOV_FR32X: case X86::CMOV_FR64: + case X86::CMOV_FR64X: case X86::CMOV_GR8: case X86::CMOV_GR16: case X86::CMOV_GR32: diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index e14a871..2289792 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -568,8 +568,14 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] in { defm _RFP80 : CMOVrr_PSEUDO; - defm _FR32 : CMOVrr_PSEUDO; - defm _FR64 : CMOVrr_PSEUDO; + let Predicates = [NoAVX512] in { + defm _FR32 : CMOVrr_PSEUDO; + defm _FR64 : CMOVrr_PSEUDO; + } + let Predicates = [HasAVX512] in { + defm _FR32X : CMOVrr_PSEUDO; + defm _FR64X : CMOVrr_PSEUDO; + } let Predicates = [NoVLX] in { defm _VR128 : CMOVrr_PSEUDO; defm _VR256 : CMOVrr_PSEUDO; diff --git a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll index a3171ed..aab74bb 100644 --- a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll +++ b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll @@ -3,6 +3,8 @@ ; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefix=CHECK --check-prefixes=FASTISEL,SSE,SSE-FASTISEL ; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -mcpu=corei7-avx | FileCheck %s --check-prefix=CHECK --check-prefixes=ISEL,AVX,AVX-ISEL ; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 -mcpu=corei7-avx | FileCheck %s --check-prefix=CHECK --check-prefixes=FASTISEL,AVX,AVX-FASTISEL +; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -mcpu=skylake-avx512 -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefixes=ISEL,AVX512,AVX512-ISEL +; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 -mcpu=skylake-avx512 -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefixes=FASTISEL,AVX512,AVX512-FASTISEL define float @select_fcmp_one_f32(float %a, float %b, float %c, float %d) { @@ -21,6 +23,13 @@ define float @select_fcmp_one_f32(float %a, float %b, float %c, float %d) { ; AVX-NEXT: vcmpneq_oqss %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vblendvps %xmm0, %xmm2, %xmm3, %xmm0 ; AVX-NEXT: retq +; +; AVX512-LABEL: select_fcmp_one_f32: +; AVX512: ## %bb.0: +; AVX512-NEXT: vcmpneq_oqss %xmm1, %xmm0, %k1 +; AVX512-NEXT: vmovss %xmm2, %xmm0, %xmm3 {%k1} +; AVX512-NEXT: vmovaps %xmm3, %xmm0 +; AVX512-NEXT: retq %1 = fcmp one float %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -42,6 +51,13 @@ define double @select_fcmp_one_f64(double %a, double %b, double %c, double %d) { ; AVX-NEXT: vcmpneq_oqsd %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vblendvpd %xmm0, %xmm2, %xmm3, %xmm0 ; AVX-NEXT: retq +; +; AVX512-LABEL: select_fcmp_one_f64: +; AVX512: ## %bb.0: +; AVX512-NEXT: vcmpneq_oqsd %xmm1, %xmm0, %k1 +; AVX512-NEXT: vmovsd %xmm2, %xmm0, %xmm3 {%k1} +; AVX512-NEXT: vmovapd %xmm3, %xmm0 +; AVX512-NEXT: retq %1 = fcmp one double %a, %b %2 = select i1 %1, double %c, double %d ret double %2 @@ -65,6 +81,24 @@ define float @select_icmp_eq_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB2_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_eq_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: sete %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_eq_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: je LBB2_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB2_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp eq i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -88,6 +122,24 @@ define float @select_icmp_ne_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB3_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_ne_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setne %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_ne_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jne LBB3_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB3_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp ne i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -111,6 +163,24 @@ define float @select_icmp_ugt_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB4_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_ugt_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: seta %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_ugt_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: ja LBB4_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB4_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp ugt i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -134,6 +204,24 @@ define float @select_icmp_uge_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB5_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_uge_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setae %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_uge_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jae LBB5_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB5_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp uge i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -157,6 +245,24 @@ define float @select_icmp_ult_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB6_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_ult_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setb %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_ult_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jb LBB6_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB6_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp ult i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -180,6 +286,24 @@ define float @select_icmp_ule_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB7_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_ule_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setbe %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_ule_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jbe LBB7_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB7_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp ule i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -203,6 +327,24 @@ define float @select_icmp_sgt_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB8_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_sgt_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setg %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_sgt_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jg LBB8_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB8_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp sgt i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -226,6 +368,24 @@ define float @select_icmp_sge_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB9_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_sge_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setge %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_sge_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jge LBB9_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB9_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp sge i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -249,6 +409,24 @@ define float @select_icmp_slt_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB10_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_slt_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setl %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_slt_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jl LBB10_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB10_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp slt i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 @@ -272,6 +450,24 @@ define float @select_icmp_sle_f32(i64 %a, i64 %b, float %c, float %d) { ; AVX-NEXT: vmovaps %xmm1, %xmm0 ; AVX-NEXT: LBB11_2: ; AVX-NEXT: retq +; +; AVX512-ISEL-LABEL: select_icmp_sle_f32: +; AVX512-ISEL: ## %bb.0: +; AVX512-ISEL-NEXT: cmpq %rsi, %rdi +; AVX512-ISEL-NEXT: setle %al +; AVX512-ISEL-NEXT: kmovd %eax, %k1 +; AVX512-ISEL-NEXT: vmovss %xmm0, %xmm0, %xmm1 {%k1} +; AVX512-ISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-ISEL-NEXT: retq +; +; AVX512-FASTISEL-LABEL: select_icmp_sle_f32: +; AVX512-FASTISEL: ## %bb.0: +; AVX512-FASTISEL-NEXT: cmpq %rsi, %rdi +; AVX512-FASTISEL-NEXT: jle LBB11_2 +; AVX512-FASTISEL-NEXT: ## %bb.1: +; AVX512-FASTISEL-NEXT: vmovaps %xmm1, %xmm0 +; AVX512-FASTISEL-NEXT: LBB11_2: +; AVX512-FASTISEL-NEXT: retq %1 = icmp sle i64 %a, %b %2 = select i1 %1, float %c, float %d ret float %2 -- 2.7.4