[InstSimplify] try harder to propagate existing NaN values through FP folds

Any undef element in a vector would trigger the whole constant
to be replaced with a canonical NaN. This propagates each
element when possible.

issue #59122

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index e2db24d..e830640 100644 (file)
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5237,8 +5237,25 @@ Value *llvm::simplifyFNegInst(Value *Op, FastMathFlags FMF,
   return ::simplifyFNegInst(Op, FMF, Q, RecursionLimit);
 }
 
+/// Try to propagate existing NaN values when possible. If not, replace the
+/// constant or elements in the constant with a canonical NaN.
 static Constant *propagateNaN(Constant *In) {
-  // If the input is a vector with undef elements, just return a default NaN.
+  if (auto *VecTy = dyn_cast<FixedVectorType>(In->getType())) {
+    unsigned NumElts = VecTy->getNumElements();
+    SmallVector<Constant *, 32> NewC(NumElts);
+    for (unsigned i = 0; i != NumElts; ++i) {
+      Constant *EltC = In->getAggregateElement(i);
+      // Poison and existing NaN elements propagate.
+      // Replace unknown or undef elements with canonical NaN.
+      if (EltC && (isa<PoisonValue>(EltC) || EltC->isNaN()))
+        NewC[i] = EltC;
+      else
+        NewC[i] = (ConstantFP::getNaN(VecTy->getElementType()));
+    }
+    return ConstantVector::get(NewC);
+  }
+
+  // It is not a fixed vector, but not a simple NaN either?
   if (!In->isNaN())
     return ConstantFP::getNaN(In->getType());
 
@@ -5273,7 +5290,13 @@ static Constant *simplifyFPOp(ArrayRef<Value *> Ops, FastMathFlags FMF,
       return PoisonValue::get(V->getType());
 
     if (isDefaultFPEnvironment(ExBehavior, Rounding)) {
-      if (IsUndef || IsNan)
+      // Undef does not propagate because undef means that all bits can take on
+      // any value. If this is undef * NaN for example, then the result values
+      // (at least the exponent bits) are limited. Assume the undef is a
+      // canonical NaN and propagate that.
+      if (IsUndef)
+        return ConstantFP::getNaN(V->getType());
+      if (IsNan)
         return propagateNaN(cast<Constant>(V));
     } else if (ExBehavior != fp::ebStrict) {
       if (IsNan)

diff --git a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
index 49acac8..31cc8e0 100644 (file)
--- a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
+++ b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
@@ -896,7 +896,7 @@ define <2 x double> @maximum_nan_op1_vec(<2 x double> %x) {
 
 define <2 x double> @minimum_nan_op0_vec_partial_undef(<2 x double> %x) {
 ; CHECK-LABEL: @minimum_nan_op0_vec_partial_undef(
-; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000DEAD00000>
 ;
   %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> <double undef, double 0x7ff8000dead00000>, <2 x double> %x)
   ret <2 x double> %r
@@ -904,7 +904,7 @@ define <2 x double> @minimum_nan_op0_vec_partial_undef(<2 x double> %x) {
 
 define <2 x double> @minimum_nan_op1_vec_partial_undef(<2 x double> %x) {
 ; CHECK-LABEL: @minimum_nan_op1_vec_partial_undef(
-; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000DEAD00000>
 ;
   %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> <double undef, double 0x7ff8000dead00000>)
   ret <2 x double> %r

diff --git a/llvm/test/Transforms/InstSimplify/fp-nan.ll b/llvm/test/Transforms/InstSimplify/fp-nan.ll
index 3773d47..1a27972 100644 (file)
--- a/llvm/test/Transforms/InstSimplify/fp-nan.ll
+++ b/llvm/test/Transforms/InstSimplify/fp-nan.ll
@@ -85,7 +85,7 @@ define <2 x half> @fdiv_nan_op1(<2 x half> %x) {
 
 define <2 x double> @fsub_nan_poison_op1(<2 x double> %x) {
 ; CHECK-LABEL: @fsub_nan_poison_op1(
-; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <2 x double> <double 0xFFFF00000000DEAD, double poison>
 ;
   %r = fsub <2 x double> %x, <double 0xFFFF00000000DEAD, double poison>
   ret <2 x double> %r
@@ -95,7 +95,7 @@ define <2 x double> @fsub_nan_poison_op1(<2 x double> %x) {
 
 define <2 x double> @frem_nan_undef_op0(<2 x double> %x) {
 ; CHECK-LABEL: @frem_nan_undef_op0(
-; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <2 x double> <double 0xFFFF00000000DEAD, double 0x7FF8000000000000>
 ;
   %r = frem <2 x double> <double 0xFFFF00000000DEAD, double undef>, %x
   ret <2 x double> %r
@@ -105,7 +105,7 @@ define <2 x double> @frem_nan_undef_op0(<2 x double> %x) {
 
 define <3 x double> @fadd_nan_poison_undef_op1(<3 x double> %x) {
 ; CHECK-LABEL: @fadd_nan_poison_undef_op1(
-; CHECK-NEXT:    ret <3 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <3 x double> <double 0xFFFF00000000DEAD, double poison, double 0x7FF8000000000000>
 ;
   %r = fadd <3 x double> %x, <double 0xFFFF00000000DEAD, double poison, double undef>
   ret <3 x double> %r