// Retrieve the VFShape that can be used to map a (scalar) function to itself,
// with VF = 1.
static VFShape getScalarShape(const CallInst &CI) {
- return VFShape::get(CI, /*EC*/ {1, false}, /*HasGlobalPredicate*/ false);
+ return VFShape::get(CI, ElementCount::getFixed(1),
+ /*HasGlobalPredicate*/ false);
}
// Retrieve the basic vectorization shape of the function, where all
inline Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
if (Scalar->isVoidTy() || VF == 1)
return Scalar;
- return VectorType::get(Scalar, {VF, isScalable});
+ return VectorType::get(Scalar, ElementCount::get(VF, isScalable));
}
/// Identify if the intrinsic is trivially vectorizable.
static VectorType *get(Type *ElementType, unsigned NumElements,
bool Scalable) {
- return VectorType::get(ElementType, {NumElements, Scalable});
+ return VectorType::get(ElementType,
+ ElementCount::get(NumElements, Scalable));
}
static VectorType *get(Type *ElementType, const VectorType *Other) {
};
inline ElementCount VectorType::getElementCount() const {
- return ElementCount(ElementQuantity, isa<ScalableVectorType>(this));
+ return ElementCount::get(ElementQuantity, isa<ScalableVectorType>(this));
}
/// Class to represent pointers.
unsigned Pointer_AddressSpace;
unsigned Struct_NumElements;
unsigned Argument_Info;
- ElementCount Vector_Width;
+ // There is no default constructor in `ElementCount`, so we need
+ // to explicitly initialize this field with a value.
+ ElementCount Vector_Width = ElementCount::getFixed(0);
};
enum ArgKind {
static IITDescriptor getVector(unsigned Width, bool IsScalable) {
IITDescriptor Result;
Result.Kind = Vector;
- Result.Vector_Width.Min = Width;
- Result.Vector_Width.Scalable = IsScalable;
+ Result.Vector_Width = ElementCount::get(Width, IsScalable);
return Result;
}
};
}
ElementCount getVectorElementCount() const {
- return { getVectorNumElements(), isScalableVector() };
+ return ElementCount::get(getVectorNumElements(), isScalableVector());
}
/// Given a vector type, return the minimum number of elements it contains.
template <typename T> struct DenseMapInfo;
class ElementCount {
+private:
+ /// Prevent code from using initializer-list contructors like
+ /// ElementCount EC = {<unsigned>, <bool>}. The static `get*`
+ /// methods below are preferred, as users should always make a
+ /// conscious choice on the type of `ElementCount` they are
+ /// requesting.
+ ElementCount(unsigned Min, bool Scalable) : Min(Min), Scalable(Scalable) {}
+
public:
+ /// No default constructor. Users should use one of the `get*`
+ /// static methods below, as they should always make a conscious
+ /// choice on the type of `ElementCount` they are requesting.
+ ElementCount() = delete;
unsigned Min; // Minimum number of vector elements.
bool Scalable; // If true, NumElements is a multiple of 'Min' determined
// at runtime rather than compile time.
- ElementCount() = default;
-
- ElementCount(unsigned Min, bool Scalable)
- : Min(Min), Scalable(Scalable) {}
-
ElementCount operator*(unsigned RHS) {
return { Min * RHS, Scalable };
}
bool operator!=(unsigned RHS) const { return !(*this == RHS); }
ElementCount NextPowerOf2() const {
- return ElementCount(llvm::NextPowerOf2(Min), Scalable);
+ return {(unsigned)llvm::NextPowerOf2(Min), Scalable};
+ }
+
+ static ElementCount getFixed(unsigned Min) { return {Min, false}; }
+ static ElementCount getScalable(unsigned Min) { return {Min, true}; }
+ static ElementCount get(unsigned Min, bool Scalable) {
+ return {Min, Scalable};
}
};
}
template <> struct DenseMapInfo<ElementCount> {
- static inline ElementCount getEmptyKey() { return {~0U, true}; }
- static inline ElementCount getTombstoneKey() { return {~0U - 1, false}; }
+ static inline ElementCount getEmptyKey() {
+ return ElementCount::getScalable(~0U);
+ }
+ static inline ElementCount getTombstoneKey() {
+ return ElementCount::getFixed(~0U - 1);
+ }
static unsigned getHashValue(const ElementCount& EltCnt) {
if (EltCnt.Scalable)
return (EltCnt.Min * 37U) - 1U;
if (auto *VTy = dyn_cast<VectorType>(Ty))
return VTy->getElementCount();
- return ElementCount(/*Min=*/1, /*Scalable=*/false);
+ return ElementCount::getFixed(/*Min=*/1);
}
} // namespace
// All vector parameters should have the same vector width.
ElementCount GEPWidth = BaseType->isVectorTy()
? cast<VectorType>(BaseType)->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
while (EatIfPresent(lltok::comma)) {
if (Lex.getKind() == lltok::MetadataVar) {
if (auto *ValVTy = dyn_cast<VectorType>(Val->getType())) {
ElementCount ValNumEl = ValVTy->getElementCount();
- if (GEPWidth != ElementCount(0, false) && GEPWidth != ValNumEl)
+ if (GEPWidth != ElementCount::getFixed(0) && GEPWidth != ValNumEl)
return Error(EltLoc,
"getelementptr vector index has a wrong number of elements");
GEPWidth = ValNumEl;
assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
"Mixing scalable and fixed vectors when copying in parts");
- ElementCount DestEltCnt;
+ Optional<ElementCount> DestEltCnt;
if (IntermediateVT.isVector())
DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
else
- DestEltCnt = ElementCount(NumIntermediates, false);
+ DestEltCnt = ElementCount::getFixed(NumIntermediates);
EVT BuiltVectorTy = EVT::getVectorVT(
- *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt);
+ *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());
if (ValueVT != BuiltVectorTy) {
if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
Val = Widened;
bool IsVectorGEP = I.getType()->isVectorTy();
ElementCount VectorElementCount =
IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
if (IsVectorGEP && !N.getValueType().isVector()) {
LLVMContext &Context = *DAG.getContext();
if (NumElts == 1)
return LegalizeKind(TypeScalarizeVector, EltVT);
- if (VT.getVectorElementCount() == ElementCount(1, true))
+ if (VT.getVectorElementCount() == ElementCount::getScalable(1))
report_fatal_error("Cannot legalize this vector");
// Try to widen vector elements until the element type is a power of two and
EVT EVT::getExtendedVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
EVT ResultVT;
- ResultVT.LLVMTy =
- VectorType::get(VT.getTypeForEVT(Context), {EC.Min, EC.Scalable});
+ ResultVT.LLVMTy = VectorType::get(VT.getTypeForEVT(Context), EC);
assert(ResultVT.isExtended() && "Type is not extended!");
return ResultVT;
}
ArrayRef<int> Mask) {
auto *V1VTy = cast<VectorType>(V1->getType());
unsigned MaskNumElts = Mask.size();
- ElementCount MaskEltCount = {MaskNumElts, isa<ScalableVectorType>(V1VTy)};
+ auto MaskEltCount =
+ ElementCount::get(MaskNumElts, isa<ScalableVectorType>(V1VTy));
Type *EltTy = V1VTy->getElementType();
// Undefined shuffle mask -> undefined value.
unsigned AS = C->getType()->getPointerAddressSpace();
Type *ReqTy = DestTy->getPointerTo(AS);
- ElementCount EltCount = {0, false};
+ auto EltCount = ElementCount::getFixed(0);
if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
EltCount = VecTy->getElementCount();
else
return getFP(V->getType(), Elts);
}
}
- return ConstantVector::getSplat({NumElts, false}, V);
+ return ConstantVector::getSplat(ElementCount::getFixed(NumElts), V);
}
Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
const Twine &Name) {
- ElementCount EC(NumElts, false);
+ auto EC = ElementCount::getFixed(NumElts);
return CreateVectorSplat(EC, V, Name);
}
// scalar types means that checking that vector lengths match also checks that
// scalars are not being converted to vectors or vectors to scalars).
ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
// Switch on the opcode provided
switch (op) {
if (SrcIsVec && DstIsVec)
return SrcEC == DstEC;
if (SrcIsVec)
- return SrcEC == ElementCount(1, false);
+ return SrcEC == ElementCount::getFixed(1);
if (DstIsVec)
- return DstEC == ElementCount(1, false);
+ return DstEC == ElementCount::getFixed(1);
return true;
}
"be an integer, floating point, or "
"pointer type.");
- ElementCount EC(NumElts, false);
+ auto EC = ElementCount::getFixed(NumElts);
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
VectorType *&Entry = ElementType->getContext()
"be an integer, floating point, or "
"pointer type.");
- ElementCount EC(MinNumElts, true);
+ auto EC = ElementCount::getScalable(MinNumElts);
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
VectorType *&Entry = ElementType->getContext()
unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8;
- EVT ByteVT = EVT::getVectorVT(Ctx, MVT::i8, { ByteSize, true });
+ EVT ByteVT =
+ EVT::getVectorVT(Ctx, MVT::i8, ElementCount::getScalable(ByteSize));
// Convert everything to the domain of EXT (i.e bytes).
SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
// FIXME: If the step is non-constant, we create the vector splat with
// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
// handle a constant vector splat.
- Value *SplatVF =
- isa<Constant>(Mul)
- ? ConstantVector::getSplat({VF, false}, cast<Constant>(Mul))
- : Builder.CreateVectorSplat(VF, Mul);
+ Value *SplatVF = isa<Constant>(Mul)
+ ? ConstantVector::getSplat(ElementCount::getFixed(VF),
+ cast<Constant>(Mul))
+ : Builder.CreateVectorSplat(VF, Mul);
Builder.restoreIP(CurrIP);
// We may need to add the step a number of times, depending on the unroll
// If we can't emit a vector call for this function, then the currently found
// cost is the cost we need to return.
NeedToScalarize = true;
- VFShape Shape = VFShape::get(*CI, {VF, false}, false /*HasGlobalPred*/);
+ VFShape Shape =
+ VFShape::get(*CI, ElementCount::getFixed(VF), false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
if (!TLI || CI->isNoBuiltin() || !VecFunc)
// incoming scalar reduction.
VectorStart = ReductionStartValue;
} else {
- Identity = ConstantVector::getSplat({VF, false}, Iden);
+ Identity = ConstantVector::getSplat(ElementCount::getFixed(VF), Iden);
// This vector is the Identity vector where the first element is the
// incoming scalar reduction.
assert(VectorF && "Can't retrieve vector intrinsic.");
} else {
// Use vector version of the function call.
- const VFShape Shape =
- VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/);
+ const VFShape Shape = VFShape::get(*CI, ElementCount::getFixed(VF),
+ false /*HasGlobalPred*/);
#ifndef NDEBUG
assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
"Can't create vector function.");
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
VFShape Shape = VFShape::get(
- *CI, {static_cast<unsigned int>(VL.size()), false /*Scalable*/},
+ *CI, ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
int IntrinsicCost =
TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
- auto Shape =
- VFShape::get(*CI, {static_cast<unsigned>(VecTy->getNumElements()), false},
- false /*HasGlobalPred*/);
+ auto Shape = VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
+ VecTy->getNumElements())),
+ false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
int LibCost = IntrinsicCost;
if (!CI->isNoBuiltin() && VecFunc) {
Function *CF;
if (!UseIntrinsic) {
- VFShape Shape = VFShape::get(
- *CI, {static_cast<unsigned>(VecTy->getNumElements()), false},
- false /*HasGlobalPred*/);
+ VFShape Shape =
+ VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
+ VecTy->getNumElements())),
+ false /*HasGlobalPred*/);
CF = VFDatabase(*CI).getVectorizedFunction(Shape);
} else {
Type *Tys[] = {FixedVectorType::get(CI->getType(), E->Scalars.size())};
Value *SplatC = IRB.CreateVectorSplat(5, ScalarC);
EXPECT_TRUE(isSplatValue(SplatC));
- Value *SplatC_SVE = IRB.CreateVectorSplat(ElementCount(5, true), ScalarC);
+ Value *SplatC_SVE =
+ IRB.CreateVectorSplat(ElementCount::getScalable(5), ScalarC);
EXPECT_TRUE(isSplatValue(SplatC_SVE));
// FIXME: Constant splat analysis does not allow undef elements.
SmallVector<VFParameter, 8> &ExpectedParams = Expected.Parameters;
void buildShape(unsigned VF, bool IsScalable, bool HasGlobalPred) {
- Shape = VFShape::get(*CI, {VF, IsScalable}, HasGlobalPred);
+ Shape = VFShape::get(*CI, ElementCount::get(VF, IsScalable), HasGlobalPred);
}
bool validParams(ArrayRef<VFParameter> Parameters) {
ASSERT_TRUE(Vnx4i32.isScalableVector());
// Create with separate llvm::ElementCount
- auto EltCnt = ElementCount(2, true);
+ auto EltCnt = ElementCount::getScalable(2);
EVT Vnx2i32 = EVT::getVectorVT(Ctx, MVT::i32, EltCnt);
ASSERT_TRUE(Vnx2i32.isScalableVector());
// Create with inline llvm::ElementCount
- EVT Vnx2i64 = EVT::getVectorVT(Ctx, MVT::i64, {2, true});
+ EVT Vnx2i64 = EVT::getVectorVT(Ctx, MVT::i64, ElementCount::getScalable(2));
ASSERT_TRUE(Vnx2i64.isScalableVector());
// Check that changing scalar types/element count works
EXPECT_EQ(EVT::getVectorVT(Ctx, MVT::i64, EltCnt / 2), MVT::nxv1i64);
// Check that float->int conversion works
- EVT Vnx2f64 = EVT::getVectorVT(Ctx, MVT::f64, {2, true});
+ EVT Vnx2f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getScalable(2));
EXPECT_EQ(Vnx2f64.changeTypeToInteger(), Vnx2i64);
// Check fields inside llvm::ElementCount
// Check that fixed-length vector types aren't scalable.
EVT V8i32 = EVT::getVectorVT(Ctx, MVT::i32, 8);
ASSERT_FALSE(V8i32.isScalableVector());
- EVT V4f64 = EVT::getVectorVT(Ctx, MVT::f64, {4, false});
+ EVT V4f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getFixed(4));
ASSERT_FALSE(V4f64.isScalableVector());
// Check that llvm::ElementCount works for fixed-length types.
LLVMContext Ctx;
Type *Int64Ty = Type::getInt64Ty(Ctx);
- VectorType *ScV8Int64Ty = VectorType::get(Int64Ty, {8, true});
+ VectorType *ScV8Int64Ty =
+ VectorType::get(Int64Ty, ElementCount::getScalable(8));
// Check that we can map a scalable IR type to an MVT
MVT Mnxv8i64 = MVT::getVT(ScV8Int64Ty);
TEST(ScalableVectorMVTsTest, VTToIRTranslation) {
LLVMContext Ctx;
- EVT Enxv4f64 = EVT::getVectorVT(Ctx, MVT::f64, {4, true});
+ EVT Enxv4f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getScalable(4));
Type *Ty = Enxv4f64.getTypeForEVT(Ctx);
VectorType *ScV4Float64Ty = cast<VectorType>(Ty);
ConstantStruct::get(StructType::create(Ctx, "OpaqueStruct"));
Constant *a =
ConstantArray::get(ArrayType::get(i32->getType(), 2), {i32, i32});
- Constant *v8i8 = ConstantVector::getSplat({8, false}, i8);
- Constant *v4f16 = ConstantVector::getSplat({4, false}, f16);
+ Constant *v8i8 = ConstantVector::getSplat(ElementCount::getFixed(8), i8);
+ Constant *v4f16 = ConstantVector::getSplat(ElementCount::getFixed(4), f16);
Constant *p0i32 =
ConstantPointerNull::get(PointerType::get(i32->getType(), 0));
Type *Int8Ty = Type::getInt8Ty(Context);
for (unsigned Min : {1, 2, 8}) {
- ElementCount ScalableEC = {Min, true};
- ElementCount FixedEC = {Min, false};
+ auto ScalableEC = ElementCount::getScalable(Min);
+ auto FixedEC = ElementCount::getFixed(Min);
for (auto EC : {ScalableEC, FixedEC}) {
for (auto *Ty : {FloatTy, Int32Ty, Int8Ty}) {
EXPECT_TRUE(match(CF32Pi, cstfp_pred_ty<always_true_pred<APFloat>>()));
EXPECT_FALSE(match(CF32Pi, cstfp_pred_ty<always_false_pred<APFloat>>()));
- ElementCount FixedEC(4, false);
- ElementCount ScalableEC(4, true);
+ auto FixedEC = ElementCount::getFixed(4);
+ auto ScalableEC = ElementCount::getScalable(4);
// Vector splat
dyn_cast<FixedVectorType>(VectorType::get(Int32Ty, V8Int32Ty));
EXPECT_VTY_EQ(V8Int32Ty, V8Int32Ty2);
- auto *V8Int16Ty =
- dyn_cast<FixedVectorType>(VectorType::get(Int16Ty, {8, false}));
+ auto *V8Int16Ty = dyn_cast<FixedVectorType>(
+ VectorType::get(Int16Ty, ElementCount::getFixed(8)));
ASSERT_NE(nullptr, V8Int16Ty);
EXPECT_EQ(V8Int16Ty->getNumElements(), 8U);
EXPECT_EQ(V8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
- ElementCount EltCnt(4, false);
+ auto EltCnt = ElementCount::getFixed(4);
auto *V4Int64Ty = dyn_cast<FixedVectorType>(VectorType::get(Int64Ty, EltCnt));
ASSERT_NE(nullptr, V4Int64Ty);
EXPECT_EQ(V4Int64Ty->getNumElements(), 4U);
dyn_cast<ScalableVectorType>(VectorType::get(Int32Ty, ScV8Int32Ty));
EXPECT_VTY_EQ(ScV8Int32Ty, ScV8Int32Ty2);
- auto *ScV8Int16Ty =
- dyn_cast<ScalableVectorType>(VectorType::get(Int16Ty, {8, true}));
+ auto *ScV8Int16Ty = dyn_cast<ScalableVectorType>(
+ VectorType::get(Int16Ty, ElementCount::getScalable(8)));
ASSERT_NE(nullptr, ScV8Int16Ty);
EXPECT_EQ(ScV8Int16Ty->getMinNumElements(), 8U);
EXPECT_EQ(ScV8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
- ElementCount EltCnt(4, true);
+ auto EltCnt = ElementCount::getScalable(4);
auto *ScV4Int64Ty =
dyn_cast<ScalableVectorType>(VectorType::get(Int64Ty, EltCnt));
ASSERT_NE(nullptr, ScV4Int64Ty);
Type *Int16Ty = Type::getInt16Ty(Ctx);
Type *Int32Ty = Type::getInt32Ty(Ctx);
- std::array<VectorType *, 8> VTys = {VectorType::get(Int16Ty, {4, true}),
- VectorType::get(Int16Ty, {4, false}),
- VectorType::get(Int16Ty, {2, true}),
- VectorType::get(Int16Ty, {2, false}),
- VectorType::get(Int32Ty, {4, true}),
- VectorType::get(Int32Ty, {4, false}),
- VectorType::get(Int32Ty, {2, true}),
- VectorType::get(Int32Ty, {2, false})};
+ std::array<VectorType *, 8> VTys = {
+ VectorType::get(Int16Ty, ElementCount::getScalable(4)),
+ VectorType::get(Int16Ty, ElementCount::getFixed(4)),
+ VectorType::get(Int16Ty, ElementCount::getScalable(2)),
+ VectorType::get(Int16Ty, ElementCount::getFixed(2)),
+ VectorType::get(Int32Ty, ElementCount::getScalable(4)),
+ VectorType::get(Int32Ty, ElementCount::getFixed(4)),
+ VectorType::get(Int32Ty, ElementCount::getScalable(2)),
+ VectorType::get(Int32Ty, ElementCount::getFixed(2))};
/*
The comparison matrix is symmetric, so we only check the upper triangle:
ConstantInt *CI = ConstantInt::get(ITy, 0);
// Valid type : freeze(<2 x i32>)
- Constant *CV = ConstantVector::getSplat({2, false}, CI);
+ Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(2), CI);
FreezeInst *FI_vec = new FreezeInst(CV);
FI_vec->insertBefore(RI);