return getGEPExpr(GEP, IndexExprs);
}
-APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) {
- uint64_t BitWidth = getTypeSizeInBits(S->getType());
- auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
- return TrailingZeros >= BitWidth
- ? APInt::getZero(BitWidth)
- : APInt::getOneBitSet(BitWidth, TrailingZeros);
- };
-
+uint32_t ScalarEvolution::getMinTrailingZerosImpl(const SCEV *S) {
switch (S->getSCEVType()) {
case scConstant:
- return cast<SCEVConstant>(S)->getAPInt();
- case scPtrToInt:
- return getConstantMultiple(cast<SCEVPtrToIntExpr>(S)->getOperand());
- case scUDivExpr:
- case scVScale:
- return APInt(BitWidth, 1);
+ return cast<SCEVConstant>(S)->getAPInt().countr_zero();
case scTruncate: {
- // Only multiples that are a power of 2 will hold after truncation.
const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(S);
- uint32_t TZ = getMinTrailingZeros(T->getOperand());
- return GetShiftedByZeros(TZ);
- }
- case scZeroExtend: {
- const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(S);
- return getConstantMultiple(Z->getOperand()).zext(BitWidth);
+ return std::min(getMinTrailingZeros(T->getOperand()),
+ (uint32_t)getTypeSizeInBits(T->getType()));
}
+ case scZeroExtend:
case scSignExtend: {
- const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(S);
- return getConstantMultiple(E->getOperand()).sext(BitWidth);
+ const SCEVIntegralCastExpr *E = cast<SCEVIntegralCastExpr>(S);
+ uint32_t OpRes = getMinTrailingZeros(E->getOperand());
+ return OpRes == getTypeSizeInBits(E->getOperand()->getType())
+ ? getTypeSizeInBits(E->getType())
+ : OpRes;
}
case scMulExpr: {
const SCEVMulExpr *M = cast<SCEVMulExpr>(S);
- if (M->hasNoUnsignedWrap()) {
- // The result is the product of all operand results.
- APInt Res = getConstantMultiple(M->getOperand(0));
- for (const SCEV *Operand : M->operands().drop_front())
- Res = Res * getConstantMultiple(Operand);
- return Res;
- }
-
- // If there are no wrap guarentees, find the trailing zeros, which is the
- // sum of trailing zeros for all its operands.
- uint32_t TZ = 0;
- for (const SCEV *Operand : M->operands())
- TZ += getMinTrailingZeros(Operand);
- return GetShiftedByZeros(TZ);
+ // The result is the sum of all operands results.
+ uint32_t SumOpRes = getMinTrailingZeros(M->getOperand(0));
+ uint32_t BitWidth = getTypeSizeInBits(M->getType());
+ for (unsigned I = 1, E = M->getNumOperands();
+ SumOpRes != BitWidth && I != E; ++I)
+ SumOpRes =
+ std::min(SumOpRes + getMinTrailingZeros(M->getOperand(I)), BitWidth);
+ return SumOpRes;
}
+ case scVScale:
+ return 0;
+ case scUDivExpr:
+ return 0;
+ case scPtrToInt:
case scAddExpr:
- case scAddRecExpr: {
- const SCEVNAryExpr *N = cast<SCEVNAryExpr>(S);
- if (N->hasNoUnsignedWrap()) {
- // The result is GCD of all operands results.
- APInt Res = getConstantMultiple(N->getOperand(0));
- for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
- Res = APIntOps::GreatestCommonDivisor(
- Res, getConstantMultiple(N->getOperand(I)));
- return Res;
- }
- }
- // If there is no unsigned wrap guarentees, fall through to find trailing
- // bits.
- LLVM_FALLTHROUGH;
+ case scAddRecExpr:
case scUMaxExpr:
case scSMaxExpr:
case scUMinExpr:
case scSMinExpr:
case scSequentialUMinExpr: {
- const SCEVNAryExpr *N = cast<SCEVNAryExpr>(S);
- // Find the trailing bits, which is the minimum of its operands.
- uint32_t TZ = getMinTrailingZeros(N->getOperand(0));
- for (const SCEV *Operand : N->operands().drop_front())
- TZ = std::min(TZ, getMinTrailingZeros(Operand));
- return GetShiftedByZeros(TZ);
+ // The result is the min of all operands results.
+ ArrayRef<const SCEV *> Ops = S->operands();
+ uint32_t MinOpRes = getMinTrailingZeros(Ops[0]);
+ for (unsigned I = 1, E = Ops.size(); MinOpRes && I != E; ++I)
+ MinOpRes = std::min(MinOpRes, getMinTrailingZeros(Ops[I]));
+ return MinOpRes;
}
case scUnknown: {
- // ask ValueTracking for known bits
const SCEVUnknown *U = cast<SCEVUnknown>(S);
- unsigned Known =
- computeKnownBits(U->getValue(), getDataLayout(), 0, &AC, nullptr, &DT)
- .countMinTrailingZeros();
- return GetShiftedByZeros(Known);
+ // For a SCEVUnknown, ask ValueTracking.
+ KnownBits Known =
+ computeKnownBits(U->getValue(), getDataLayout(), 0, &AC, nullptr, &DT);
+ return Known.countMinTrailingZeros();
}
case scCouldNotCompute:
llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
llvm_unreachable("Unknown SCEV kind!");
}
-APInt ScalarEvolution::getConstantMultiple(const SCEV *S) {
- auto I = ConstantMultipleCache.find(S);
- if (I != ConstantMultipleCache.end())
+uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S) {
+ auto I = MinTrailingZerosCache.find(S);
+ if (I != MinTrailingZerosCache.end())
return I->second;
- APInt Result = getConstantMultipleImpl(S);
- auto InsertPair = ConstantMultipleCache.insert({S, Result});
+ uint32_t Result = getMinTrailingZerosImpl(S);
+ auto InsertPair = MinTrailingZerosCache.insert({S, Result});
assert(InsertPair.second && "Should insert a new key");
return InsertPair.first->second;
}
-APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
- APInt Multiple = getConstantMultiple(S);
- return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
-}
-
-uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S) {
- return std::min(getConstantMultiple(S).countTrailingZeros(),
- (unsigned)getTypeSizeInBits(S->getType()));
-}
-
/// Helper method to assign a range to V from metadata present in the IR.
static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
if (Instruction *I = dyn_cast<Instruction>(V))
// If the value has known zeros, the maximum value will have those known zeros
// as well.
- if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
- APInt Multiple = getNonZeroConstantMultiple(S);
- APInt Remainder = APInt::getMaxValue(BitWidth).urem(Multiple);
- if (!Remainder.isZero())
+ uint32_t TZ = getMinTrailingZeros(S);
+ if (TZ != 0) {
+ if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED)
ConservativeResult =
ConstantRange(APInt::getMinValue(BitWidth),
- APInt::getMaxValue(BitWidth) - Remainder + 1);
- }
- else {
- uint32_t TZ = getMinTrailingZeros(S);
- if (TZ != 0) {
+ APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
+ else
ConservativeResult = ConstantRange(
APInt::getSignedMinValue(BitWidth),
APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
- }
}
switch (S->getSCEVType()) {
};
const SCEVConstant *TC = dyn_cast<SCEVConstant>(TCExpr);
- if (!TC) {
- APInt Multiple = getNonZeroConstantMultiple(TCExpr);
- return Multiple.getActiveBits() > 32 ? 1 : *Multiple.getRawData();
- }
+ if (!TC)
+ // Attempt to factor more general cases. Returns the greatest power of
+ // two divisor.
+ return GetSmallMultiple(getMinTrailingZeros(TCExpr));
ConstantInt *Result = TC->getValue();
assert(Result && "SCEVConstant expected to have non-null ConstantInt");
SignedRanges.clear();
ExprValueMap.clear();
HasRecMap.clear();
- ConstantMultipleCache.clear();
+ MinTrailingZerosCache.clear();
PredicatedSCEVRewrites.clear();
FoldCache.clear();
FoldCacheUser.clear();
PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
PendingPhiRanges(std::move(Arg.PendingPhiRanges)),
PendingMerges(std::move(Arg.PendingMerges)),
- ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
+ MinTrailingZerosCache(std::move(Arg.MinTrailingZerosCache)),
BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
PredicatedBackedgeTakenCounts(
std::move(Arg.PredicatedBackedgeTakenCounts)),
UnsignedRanges.erase(S);
SignedRanges.erase(S);
HasRecMap.erase(S);
- ConstantMultipleCache.erase(S);
+ MinTrailingZerosCache.erase(S);
if (auto *AR = dyn_cast<SCEVAddRecExpr>(S)) {
UnsignedWrapViaInductionTried.erase(AR);
}
}
}
-
- // Verify that ConstantMultipleCache computations are correct.
- for (auto [S, Multiple] : ConstantMultipleCache) {
- APInt RecomputedMultiple = SE2.getConstantMultipleImpl(S);
- if (Multiple != RecomputedMultiple) {
- dbgs() << "Incorrect computation in ConstantMultipleCache for " << *S
- << " : Expected " << RecomputedMultiple << " but got " << Multiple
- << "!\n";
- std::abort();
- }
- }
}
bool ScalarEvolution::invalidate(
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
- ; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>,verify<scalar-evolution>" 2>&1 | FileCheck %s
- ; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>,verify<scalar-evolution>" -scev-range-iter-threshold=1 2>&1 | FileCheck %s
+ ; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>" 2>&1 | FileCheck %s
+ ; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>" -scev-range-iter-threshold=1 2>&1 | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64"
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
; CHECK-NEXT: --> {0,+,6}<nuw><nsw><%loop> U: [0,-2147483648) S: [0,2147483647) Exits: (6 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 6) + (1 umin %n))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %iv.inc = add nsw i32 %iv, 6
-; CHECK-NEXT: --> {6,+,6}<nuw><%loop> U: [6,-3) S: [-2147483648,2147483647) Exits: (6 + (6 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 6) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT: --> {6,+,6}<nuw><%loop> U: [6,-1) S: [-2147483648,2147483647) Exits: (6 + (6 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 6) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: Determining loop execution counts for: @add_6
; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 6) + (1 umin %n))
; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 715827882
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
; CHECK-NEXT: --> {0,+,7}<nuw><nsw><%loop> U: [0,-2147483648) S: [0,-2147483648) Exits: (7 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 7) + (1 umin %n))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %iv.inc = add nsw i32 %iv, 7
-; CHECK-NEXT: --> {7,+,7}<nuw><%loop> U: [7,-3) S: [7,0) Exits: (7 + (7 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 7) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT: --> {7,+,7}<nuw><%loop> U: [7,0) S: [7,0) Exits: (7 + (7 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 7) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: Determining loop execution counts for: @add_7
; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 7) + (1 umin %n))
; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 613566756
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
; CHECK-NEXT: --> {0,+,9}<nuw><nsw><%loop> U: [0,-2147483648) S: [0,-2147483648) Exits: (9 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 9) + (1 umin %n))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %iv.inc = add nsw i32 %iv, 9
-; CHECK-NEXT: --> {9,+,9}<nuw><%loop> U: [9,-3) S: [9,0) Exits: (9 + (9 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 9) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT: --> {9,+,9}<nuw><%loop> U: [9,0) S: [9,0) Exits: (9 + (9 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 9) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: Determining loop execution counts for: @add_9
; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 9) + (1 umin %n))
; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 477218588
; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
; CHECK-NEXT: --> {0,+,10}<nuw><nsw><%loop> U: [0,-2147483648) S: [0,2147483647) Exits: (10 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 10) + (1 umin %n))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: %iv.inc = add nsw i32 %iv, 10
-; CHECK-NEXT: --> {10,+,10}<nuw><%loop> U: [10,-5) S: [-2147483648,2147483647) Exits: (10 + (10 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 10) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT: --> {10,+,10}<nuw><%loop> U: [10,-1) S: [-2147483648,2147483647) Exits: (10 + (10 * ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 10) + (1 umin %n)))) LoopDispositions: { %loop: Computable }
; CHECK-NEXT: Determining loop execution counts for: @add_10
; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n))<nuw><nsw> + %n) /u 10) + (1 umin %n))
; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 429496729
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes='print<scalar-evolution>,verify<scalar-evolution>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<scalar-evolution>' -disable-output %s 2>&1 | FileCheck %s
; Test trip multiples with functions that look like:
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 3
+; CHECK: Loop %for.body: Trip multiple is 1
;
entry:
%rem = urem i32 %num, 3
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 5
+; CHECK: Loop %for.body: Trip multiple is 1
;
entry:
%rem = urem i32 %num, 5
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 6
+; CHECK: Loop %for.body: Trip multiple is 2
;
entry:
%rem = urem i32 %num, 6
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 7
+; CHECK: Loop %for.body: Trip multiple is 1
;
entry:
%rem = urem i32 %num, 7
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 9
+; CHECK: Loop %for.body: Trip multiple is 1
;
entry:
%rem = urem i32 %num, 9
; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num)
; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num)
; CHECK-NEXT: Predicates:
-; CHECK: Loop %for.body: Trip multiple is 10
+; CHECK: Loop %for.body: Trip multiple is 2
;
entry:
%rem = urem i32 %num, 10
projects / platform / upstream / llvm.git / commitdiff