From 3681a7dd7818aa93956fe42d37e97da6882f23ae Mon Sep 17 00:00:00 2001 From: David Truby Date: Fri, 14 Jul 2023 17:02:44 +0100 Subject: [PATCH] [flang] Use libm functions for complex operations by default This patch changes the default lowering for complex operations to use the more accurate libm operations as opposed to the mlir complex operations. This is necessary due to precision issues in the mlir complex dialect that cause failures in e.g. the LAPACK tests. The mlir complex dialect lowering will still be used when `-fapprox-func` is set (and by extension `-ffast-math` and `-Ofast`) Differential Revision: https://reviews.llvm.org/D155310 --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 +++++--- flang/test/Lower/HLFIR/binary-ops.f90 | 6 ++---- flang/test/Lower/Intrinsics/abs.f90 | 5 +++-- flang/test/Lower/Intrinsics/exp.f90 | 11 +++++++---- flang/test/Lower/Intrinsics/log.f90 | 11 +++++++---- flang/test/Lower/math-lowering.f90 | 10 ++-------- flang/test/Lower/power-operator.f90 | 5 +++-- flang/test/Lower/sqrt.f90 | 5 +++-- flang/test/Lower/trigonometric-intrinsics.f90 | 3 ++- 9 files changed, 34 insertions(+), 30 deletions(-) diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 74a6e1c..7d1a9ff 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -733,8 +733,9 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc, mlir::FunctionType mathLibFuncType, llvm::ArrayRef args) { mlir::Value result; - if (disableMlirComplex || - (mathRuntimeVersion == preciseVersion && !mathLibFuncName.empty())) { + bool hasApproxFunc = mlir::arith::bitEnumContainsAny( + builder.getFastMathFlags(), mlir::arith::FastMathFlags::afn); + if ((disableMlirComplex || !hasApproxFunc) && !mathLibFuncName.empty()) { result = genLibCall(builder, loc, mathLibFuncName, mathLibFuncType, args); LLVM_DEBUG(result.dump(); llvm::dbgs() << "\n"); return result; @@ -1204,8 +1205,9 @@ searchMathOperation(fir::FirOpBuilder &builder, llvm::StringRef name, for (auto iter = range.first; iter != range.second && iter; ++iter) { const auto &impl = *iter; auto implType = impl.typeGenerator(builder.getContext(), builder); - if (funcType == implType) + if (funcType == implType) { return &impl; // exact match + } FunctionDistance distance(funcType, implType); if (distance.isSmallerThan(bestMatchDistance)) { diff --git a/flang/test/Lower/HLFIR/binary-ops.f90 b/flang/test/Lower/HLFIR/binary-ops.f90 index 3809f0d..8db6da3 100644 --- a/flang/test/Lower/HLFIR/binary-ops.f90 +++ b/flang/test/Lower/HLFIR/binary-ops.f90 @@ -168,10 +168,8 @@ end subroutine ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref> ! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref> -! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.complex<4>) -> complex -! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_7]] : (!fir.complex<4>) -> complex -! CHECK: %[[VAL_10:.*]] = complex.pow %[[VAL_8]], %[[VAL_9]] : complex -! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (complex) -> !fir.complex<4> +! CHECK: %[[VAL_8:.*]] = fir.call @cpowf(%[[VAL_6]], %[[VAL_7]]) fastmath : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4> + subroutine real_to_int_power(x, y, z) real :: x, y diff --git a/flang/test/Lower/Intrinsics/abs.f90 b/flang/test/Lower/Intrinsics/abs.f90 index 9f67757..896a146 100644 --- a/flang/test/Lower/Intrinsics/abs.f90 +++ b/flang/test/Lower/Intrinsics/abs.f90 @@ -1,9 +1,10 @@ -! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" ! RUN: bbc -emit-fir --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: bbc --disable-mlir-complex -emit-fir %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" -! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm --disable-mlir-complex %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" +! RUN: %flang_fc1 -fapprox-func -emit-fir %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-FAST" ! Test abs intrinsic for various types (int, float, complex) diff --git a/flang/test/Lower/Intrinsics/exp.f90 b/flang/test/Lower/Intrinsics/exp.f90 index eb42235..8188ba3 100644 --- a/flang/test/Lower/Intrinsics/exp.f90 +++ b/flang/test/Lower/Intrinsics/exp.f90 @@ -1,7 +1,8 @@ -! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" ! RUN: bbc -emit-fir --math-runtime=precise -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: bbc -emit-fir --disable-mlir-complex -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" -! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" +! RUN: %flang_fc1 -fapprox-func -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-FAST" ! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics -mllvm --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics -mllvm --disable-mlir-complex %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" @@ -55,7 +56,8 @@ end subroutine ! CHECK: %[[RESULT64_OUTLINE:.*]] = math.exp %[[ARG64_OUTLINE]] fastmath : f64 ! CHECK: return %[[RESULT64_OUTLINE]] : f64 -! CMPLX-LABEL: private @fir.exp.contract.z4.z4 +! CMPLX-FAST-LABEL: private @fir.exp.contract_afn.z4.z4 +! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z4.z4 ! CMPLX-SAME: (%[[ARG32_OUTLINE:.*]]: !fir.complex<4>) -> !fir.complex<4> ! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex ! CMPLX-FAST: %[[E:.*]] = complex.exp %[[C]] : complex @@ -63,7 +65,8 @@ end subroutine ! CMPLX-PRECISE: %[[RESULT32_OUTLINE:.*]] = fir.call @cexpf(%[[ARG32_OUTLINE]]) fastmath : (!fir.complex<4>) -> !fir.complex<4> ! CMPLX: return %[[RESULT32_OUTLINE]] : !fir.complex<4> -! CMPLX-LABEL: private @fir.exp.contract.z8.z8 +! CMPLX-FAST-LABEL: private @fir.exp.contract_afn.z8.z8 +! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z8.z8 ! CMPLX-SAME: (%[[ARG64_OUTLINE:.*]]: !fir.complex<8>) -> !fir.complex<8> ! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex ! CMPLX-FAST: %[[E:.*]] = complex.exp %[[C]] : complex diff --git a/flang/test/Lower/Intrinsics/log.f90 b/flang/test/Lower/Intrinsics/log.f90 index e3b9ba8..e39972d 100644 --- a/flang/test/Lower/Intrinsics/log.f90 +++ b/flang/test/Lower/Intrinsics/log.f90 @@ -1,9 +1,10 @@ -! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" ! RUN: bbc -emit-fir --math-runtime=precise -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: bbc -emit-fir --disable-mlir-complex -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" -! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-FAST" +! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX,CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics -mllvm --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics -mllvm --disable-mlir-complex %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-PRECISE" +! RUN: %flang_fc1 -fapprox-func -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX,CMPLX-FAST" ! CHECK-LABEL: log_testr ! CHECK-SAME: (%[[AREF:.*]]: !fir.ref {{.*}}, %[[BREF:.*]]: !fir.ref {{.*}}) @@ -75,7 +76,8 @@ end subroutine ! CHECK: %[[RESULT64_OUTLINE:.*]] = math.log %[[ARG64_OUTLINE]] fastmath : f64 ! CHECK: return %[[RESULT64_OUTLINE]] : f64 -! CMPLX-LABEL: private @fir.log.contract.z4.z4 +! CMPLX-FAST-LABEL: private @fir.log.contract_afn.z4.z4 +! CMPLX-PRECISE-LABEL: private @fir.log.contract.z4.z4 ! CMPLX-SAME: (%[[ARG32_OUTLINE:.*]]: !fir.complex<4>) -> !fir.complex<4> ! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex ! CMPLX-FAST: %[[E:.*]] = complex.log %[[C]] : complex @@ -83,7 +85,8 @@ end subroutine ! CMPLX-PRECISE: %[[RESULT32_OUTLINE:.*]] = fir.call @clogf(%[[ARG32_OUTLINE]]) fastmath : (!fir.complex<4>) -> !fir.complex<4> ! CMPLX: return %[[RESULT32_OUTLINE]] : !fir.complex<4> -! CMPLX-LABEL: private @fir.log.contract.z8.z8 +! CMPLX-FAST-LABEL: private @fir.log.contract_afn.z8.z8 +! CMPLX-PRECISE-LABEL: private @fir.log.contract.z8.z8 ! CMPLX-SAME: (%[[ARG64_OUTLINE:.*]]: !fir.complex<8>) -> !fir.complex<8> ! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex ! CMPLX-FAST: %[[E:.*]] = complex.log %[[C]] : complex diff --git a/flang/test/Lower/math-lowering.f90 b/flang/test/Lower/math-lowering.f90 index b1ee05a..973cbcf 100644 --- a/flang/test/Lower/math-lowering.f90 +++ b/flang/test/Lower/math-lowering.f90 @@ -41,21 +41,15 @@ function test_complex4(c) complex(4) :: c, test_complex4 test_complex4 = abs(c) end function - ! ALL-LABEL: @_QPtest_complex4 -! FAST: {{%[A-Za-z0-9._]+}} = complex.abs {{%[A-Za-z0-9._]+}} : complex -! RELAXED: {{%[A-Za-z0-9._]+}} = complex.abs {{%[A-Za-z0-9._]+}} : complex -! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @cabsf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> f32 +! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabsf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> f32 function test_complex8(c) complex(8) :: c, test_complex8 test_complex8 = abs(c) end function - ! ALL-LABEL: @_QPtest_complex8 -! FAST: {{%[A-Za-z0-9._]+}} = complex.abs {{%[A-Za-z0-9._]+}} : complex -! RELAXED: {{%[A-Za-z0-9._]+}} = complex.abs {{%[A-Za-z0-9._]+}} : complex -! PRECISE: {{%[A-Za-z0-9._]+}} = fir.call @cabs({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> f64 +! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabs({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> f64 ! PRECISE-DAG: func.func private @fabsf(f32) -> f32 attributes {fir.bindc_name = "fabsf", fir.runtime} ! PRECISE-DAG: func.func private @fabs(f64) -> f64 attributes {fir.bindc_name = "fabs", fir.runtime} diff --git a/flang/test/Lower/power-operator.f90 b/flang/test/Lower/power-operator.f90 index 67b93d3..7b80bf4 100644 --- a/flang/test/Lower/power-operator.f90 +++ b/flang/test/Lower/power-operator.f90 @@ -1,7 +1,8 @@ -! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,FAST" +! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,PRECISE" ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="PRECISE" ! RUN: bbc --disable-mlir-complex -emit-fir %s -o - | FileCheck %s --check-prefixes="PRECISE" -! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,FAST" +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,PRECISE" +! RUN: %flang_fc1 -fapprox-func -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,FAST" ! RUN: %flang_fc1 -emit-fir -mllvm --math-runtime=precise %s -o - | FileCheck %s --check-prefixes="PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm --disable-mlir-complex %s -o - | FileCheck %s --check-prefixes="PRECISE" diff --git a/flang/test/Lower/sqrt.f90 b/flang/test/Lower/sqrt.f90 index e55911a4..91cdb91 100644 --- a/flang/test/Lower/sqrt.f90 +++ b/flang/test/Lower/sqrt.f90 @@ -1,7 +1,8 @@ -! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-FAST" +! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-PRECISE" ! RUN: bbc --math-runtime=precise -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-PRECISE" ! RUN: bbc --disable-mlir-complex -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-PRECISE" -! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-FAST" +! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-PRECISE" +! RUN: %flang_fc1 -fapprox-func -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-FAST" ! RUN: %flang_fc1 -emit-fir -mllvm --math-runtime=precise -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm --disable-mlir-complex -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-PRECISE" diff --git a/flang/test/Lower/trigonometric-intrinsics.f90 b/flang/test/Lower/trigonometric-intrinsics.f90 index 139bf10..44a6b71 100644 --- a/flang/test/Lower/trigonometric-intrinsics.f90 +++ b/flang/test/Lower/trigonometric-intrinsics.f90 @@ -1,6 +1,7 @@ -! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-FAST" +! RUN: bbc -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CHECK,CMPLX-PRECISE" ! RUN: bbc --math-runtime=precise -emit-fir -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-PRECISE" ! RUN: %flang_fc1 -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s +! RUN: %flang_fc1 -fapprox-func -emit-fir -mllvm -outline-intrinsics %s -o - | FileCheck %s --check-prefixes="CMPLX-FAST" ! CHECK-LABEL: tan_testr subroutine tan_testr(a, b) -- 2.7.4