[SelectionDAG][GlobalISel] Move even divisor optimization for division by constant...

I've added a bool to UnsignedDivideUsingMagic so we can continue
testing it in the unit test with and without this optimization in
the unit test.

This is a step towards supporting "uncooperative" odd divisors.
See https://ridiculousfish.com/blog/posts/labor-of-division-episode-iii.html

Reviewed By: lebedev.ri

Differential Revision: https://reviews.llvm.org/D140924

diff --git a/llvm/include/llvm/Support/DivisionByConstantInfo.h b/llvm/include/llvm/Support/DivisionByConstantInfo.h
index 7d01613..464e412 100644 (file)
--- a/llvm/include/llvm/Support/DivisionByConstantInfo.h
+++ b/llvm/include/llvm/Support/DivisionByConstantInfo.h
@@ -26,11 +26,13 @@ struct SignedDivisionByConstantInfo {
 
 /// Magic data for optimising unsigned division by a constant.
 struct UnsignedDivisionByConstantInfo {
-  static UnsignedDivisionByConstantInfo get(const APInt &D,
-                                            unsigned LeadingZeros = 0);
+  static UnsignedDivisionByConstantInfo
+  get(const APInt &D, unsigned LeadingZeros = 0,
+      bool AllowEvenDivisorOptimization = true);
   APInt Magic;          ///< magic number
   bool IsAdd;           ///< add indicator
   unsigned ShiftAmount; ///< shift amount
+  unsigned PreShift;    ///< pre-shift amount
 };
 
 } // namespace llvm

diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index d9ae99f..89ebcdc 100644 (file)
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -4952,23 +4952,15 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) {
         UnsignedDivisionByConstantInfo::get(Divisor);
     unsigned PreShift = 0, PostShift = 0;
 
-    // If the divisor is even, we can avoid using the expensive fixup by
-    // shifting the divided value upfront.
-    if (magics.IsAdd && !Divisor[0]) {
-      PreShift = Divisor.countTrailingZeros();
-      // Get magic number for the shifted divisor.
-      magics =
-          UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
-      assert(!magics.IsAdd && "Should use cheap fixup now");
-    }
-
     unsigned SelNPQ;
     if (!magics.IsAdd || Divisor.isOneValue()) {
       assert(magics.ShiftAmount < Divisor.getBitWidth() &&
              "We shouldn't generate an undefined shift!");
       PostShift = magics.ShiftAmount;
+      PreShift = magics.PreShift;
       SelNPQ = false;
     } else {
+      assert(magics.PreShift == 0 && "Unexpected pre-shift");
       PostShift = magics.ShiftAmount - 1;
       SelNPQ = true;
     }

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ad7ea4f..ebe7716 100644 (file)
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6046,23 +6046,15 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
         UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros);
     unsigned PreShift = 0, PostShift = 0;
 
-    // If the divisor is even, we can avoid using the expensive fixup by
-    // shifting the divided value upfront.
-    if (magics.IsAdd && !Divisor[0]) {
-      PreShift = Divisor.countTrailingZeros();
-      // Get magic number for the shifted divisor.
-      magics = UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift),
-                                                   PreShift + LeadingZeros);
-      assert(!magics.IsAdd && "Should use cheap fixup now");
-    }
-
     unsigned SelNPQ;
     if (!magics.IsAdd || Divisor.isOne()) {
       assert(magics.ShiftAmount < Divisor.getBitWidth() &&
              "We shouldn't generate an undefined shift!");
       PostShift = magics.ShiftAmount;
+      PreShift = magics.PreShift;
       SelNPQ = false;
     } else {
+      assert(magics.PreShift == 0 && "Unexpected pre-shift");
       PostShift = magics.ShiftAmount - 1;
       SelNPQ = true;
     }

diff --git a/llvm/lib/Support/DivisionByConstantInfo.cpp b/llvm/lib/Support/DivisionByConstantInfo.cpp
index b290760..b795c09 100644 (file)
--- a/llvm/lib/Support/DivisionByConstantInfo.cpp
+++ b/llvm/lib/Support/DivisionByConstantInfo.cpp
@@ -71,7 +71,8 @@ SignedDivisionByConstantInfo SignedDivisionByConstantInfo::get(const APInt &D) {
 /// LeadingZeros can be used to simplify the calculation if the upper bits
 /// of the divided value are known zero.
 UnsignedDivisionByConstantInfo
-UnsignedDivisionByConstantInfo::get(const APInt &D, unsigned LeadingZeros) {
+UnsignedDivisionByConstantInfo::get(const APInt &D, unsigned LeadingZeros,
+                                    bool AllowEvenDivisorOptimization) {
   assert(!D.isZero() && "Precondition violation.");
   assert(D.getBitWidth() > 1 && "Does not work at smaller bitwidths.");
 
@@ -129,8 +130,20 @@ UnsignedDivisionByConstantInfo::get(const APInt &D, unsigned LeadingZeros) {
     Delta -= R2;
   } while (P < D.getBitWidth() * 2 &&
            (Q1.ult(Delta) || (Q1 == Delta && R1.isZero())));
+
+  if (Retval.IsAdd && !D[0] && AllowEvenDivisorOptimization) {
+    unsigned PreShift = D.countTrailingZeros();
+    APInt ShiftedD = D.lshr(PreShift);
+    Retval =
+        UnsignedDivisionByConstantInfo::get(ShiftedD, LeadingZeros + PreShift);
+    assert(Retval.IsAdd == 0 && Retval.PreShift == 0);
+    Retval.PreShift = PreShift;
+    return Retval;
+  }
+
   Retval.Magic = std::move(Q2);             // resulting magic number
   ++Retval.Magic;
   Retval.ShiftAmount = P - D.getBitWidth(); // resulting shift
+  Retval.PreShift = 0;
   return Retval;
 }

diff --git a/llvm/unittests/Support/DivisionByConstantTest.cpp b/llvm/unittests/Support/DivisionByConstantTest.cpp
index 72babee..90ae918 100644 (file)
--- a/llvm/unittests/Support/DivisionByConstantTest.cpp
+++ b/llvm/unittests/Support/DivisionByConstantTest.cpp
@@ -108,32 +108,23 @@ APInt UnsignedDivideUsingMagic(const APInt &Numerator, const APInt &Divisor,
     // Clip to the number of leading zeros in the divisor.
     LeadingZeros = std::min(LeadingZeros, Divisor.countLeadingZeros());
     if (LeadingZeros > 0) {
-      Magics = UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros);
+      Magics = UnsignedDivisionByConstantInfo::get(
+          Divisor, LeadingZeros, AllowEvenDivisorOptimization);
       assert(!Magics.IsAdd && "Should use cheap fixup now");
     }
   }
 
   unsigned PreShift = 0;
-  if (AllowEvenDivisorOptimization) {
-    // If the divisor is even, we can avoid using the expensive fixup by
-    // shifting the divided value upfront.
-    if (Magics.IsAdd && !Divisor[0]) {
-      PreShift = Divisor.countTrailingZeros();
-      // Get magic number for the shifted divisor.
-      Magics =
-          UnsignedDivisionByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
-      assert(!Magics.IsAdd && "Should use cheap fixup now");
-    }
-  }
-
   unsigned PostShift = 0;
   bool UseNPQ = false;
   if (!Magics.IsAdd || Divisor.isOne()) {
     assert(Magics.ShiftAmount < Divisor.getBitWidth() &&
            "We shouldn't generate an undefined shift!");
+    PreShift = Magics.PreShift;
     PostShift = Magics.ShiftAmount;
     UseNPQ = false;
   } else {
+    assert(Magics.PreShift == 0 && "Unexpected pre-shift");
     PostShift = Magics.ShiftAmount - 1;
     assert(PostShift < Divisor.getBitWidth() &&
            "We shouldn't generate an undefined shift!");