[X86][SSE] Begin removing vector CTTZ custom lowering and use LegalizeDAG instead.

Adds CTTZ vector legalization support and begins the removal of the X86/SSE custom lowering.

llvm-svn: 344453

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5602511..884d717 100644 (file)
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2794,7 +2794,7 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
     // This trivially expands to CTTZ.
     return DAG.getNode(ISD::CTTZ, dl, VT, Op);
   case ISD::CTTZ: {
-    unsigned Len = VT.getSizeInBits();
+    unsigned Len = VT.getScalarSizeInBits();
 
     if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
       EVT SetCCVT = getSetCCResultType(VT);

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 8cc37b5..58d86e8 100644 (file)
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -129,7 +129,7 @@ class VectorLegalizer {
   SDValue ExpandFSUB(SDValue Op);
   SDValue ExpandBITREVERSE(SDValue Op);
   SDValue ExpandCTLZ(SDValue Op);
-  SDValue ExpandCTTZ_ZERO_UNDEF(SDValue Op);
+  SDValue ExpandCTTZ(SDValue Op);
   SDValue ExpandStrictFPOp(SDValue Op);
 
   /// Implements vector promotion.
@@ -717,8 +717,9 @@ SDValue VectorLegalizer::Expand(SDValue Op) {
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF:
     return ExpandCTLZ(Op);
+  case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF:
-    return ExpandCTTZ_ZERO_UNDEF(Op);
+    return ExpandCTTZ(Op);
   case ISD::STRICT_FADD:
   case ISD::STRICT_FSUB:
   case ISD::STRICT_FMUL:
@@ -1094,8 +1095,9 @@ SDValue VectorLegalizer::ExpandCTLZ(SDValue Op) {
   return DAG.UnrollVectorOp(Op.getNode());
 }
 
-SDValue VectorLegalizer::ExpandCTTZ_ZERO_UNDEF(SDValue Op) {
+SDValue VectorLegalizer::ExpandCTTZ(SDValue Op) {
   EVT VT = Op.getValueType();
+  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
 
   // If the non-ZERO_UNDEF version is supported we can use that instead.
   if (TLI.isOperationLegalOrCustom(ISD::CTTZ, VT)) {
@@ -1103,6 +1105,16 @@ SDValue VectorLegalizer::ExpandCTTZ_ZERO_UNDEF(SDValue Op) {
     return DAG.getNode(ISD::CTTZ, DL, VT, Op.getOperand(0));
   }
 
+  // If we have the appropriate vector bit operations, it is better to use them
+  // than unrolling and expanding each component.
+  if (isPowerOf2_32(NumBitsPerElt) &&
+      (TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) ||
+       TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) &&
+      TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
+      TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
+      TLI.isOperationLegalOrCustomOrPromote(ISD::XOR, VT))
+    return Op;
+
   // Otherwise go ahead and unroll.
   return DAG.UnrollVectorOp(Op.getNode());
 }

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 18c5f60..1411cf1 100644 (file)
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -23004,22 +23004,21 @@ static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
     if (VT.is256BitVector() && !Subtarget.hasInt256())
       return Lower256IntUnary(Op, DAG);
 
-    // Tmp = ~x & (x - 1)
-    SDValue One = DAG.getConstant(1, dl, VT);
-    SDValue Tmp = DAG.getNode(ISD::AND, dl, VT, DAG.getNOT(dl, N0, VT),
-                              DAG.getNode(ISD::SUB, dl, VT, N0, One));
-
     // cttz(x) = width - ctlz(~x & (x - 1))
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     if (TLI.isOperationLegal(ISD::CTLZ, VT) &&
         !TLI.isOperationLegal(ISD::CTPOP, VT)) {
+      SDValue One = DAG.getConstant(1, dl, VT);
       SDValue Width = DAG.getConstant(NumBits, dl, VT);
-      return DAG.getNode(ISD::SUB, dl, VT, Width,
-                         DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
+      return DAG.getNode(
+          ISD::SUB, dl, VT, Width,
+          DAG.getNode(ISD::CTLZ, dl, VT,
+                      DAG.getNode(ISD::AND, dl, VT, DAG.getNOT(dl, N0, VT),
+                                  DAG.getNode(ISD::SUB, dl, VT, N0, One))));
     }
 
-    // cttz(x) = ctpop(~x & (x - 1))
-    return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
+    // Else leave it to the legalizer.
+    return SDValue();
   }
 
   assert(Op.getOpcode() == ISD::CTTZ &&