[DataLayout] Use separate vectors to store alignment (NFC)

Instead of storing alignment for integers, floats, vectors and
structs in a single vector with a type tag, store them in
separate vectors instead. This makes the alignment lookup faster,
as we don't have to scan over irrelevant alignment entries.

diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index fbfbf77..f72f918 100644 (file)
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -52,7 +52,6 @@ class Value;
 
 /// Enum used to categorize the alignment types stored by LayoutAlignElem
 enum AlignTypeEnum {
-  INVALID_ALIGN = 0,
   INTEGER_ALIGN = 'i',
   VECTOR_ALIGN = 'v',
   FLOAT_ALIGN = 'f',
@@ -66,20 +65,17 @@ enum AlignTypeEnum {
 
 /// Layout alignment element.
 ///
-/// Stores the alignment data associated with a given alignment type (integer,
-/// vector, float) and type bit width.
+/// Stores the alignment data associated with a given type bit width.
 ///
 /// \note The unusual order of elements in the structure attempts to reduce
 /// padding and make the structure slightly more cache friendly.
 struct LayoutAlignElem {
-  /// Alignment type from \c AlignTypeEnum
-  unsigned AlignType : 8;
-  unsigned TypeBitWidth : 24;
+  uint32_t TypeBitWidth;
   Align ABIAlign;
   Align PrefAlign;
 
-  static LayoutAlignElem get(AlignTypeEnum align_type, Align abi_align,
-                             Align pref_align, uint32_t bit_width);
+  static LayoutAlignElem get(Align ABIAlign, Align PrefAlign,
+                             uint32_t BitWidth);
 
   bool operator==(const LayoutAlignElem &rhs) const;
 };
@@ -147,17 +143,11 @@ private:
 
   /// Primitive type alignment data. This is sorted by type and bit
   /// width during construction.
-  using AlignmentsTy = SmallVector<LayoutAlignElem, 16>;
-  AlignmentsTy Alignments;
-
-  AlignmentsTy::const_iterator
-  findAlignmentLowerBound(AlignTypeEnum AlignType, uint32_t BitWidth) const {
-    return const_cast<DataLayout *>(this)->findAlignmentLowerBound(AlignType,
-                                                                   BitWidth);
-  }
-
-  AlignmentsTy::iterator
-  findAlignmentLowerBound(AlignTypeEnum AlignType, uint32_t BitWidth);
+  using AlignmentsTy = SmallVector<LayoutAlignElem, 4>;
+  AlignmentsTy IntAlignments;
+  AlignmentsTy FloatAlignments;
+  AlignmentsTy VectorAlignments;
+  LayoutAlignElem StructAlignment;
 
   /// The string representation used to create this DataLayout
   std::string StringRepresentation;
@@ -176,8 +166,8 @@ private:
 
   /// Attempts to set the alignment of the given type. Returns an error
   /// description on failure.
-  Error setAlignment(AlignTypeEnum align_type, Align abi_align,
-                     Align pref_align, uint32_t bit_width);
+  Error setAlignment(AlignTypeEnum AlignType, Align ABIAlign, Align PrefAlign,
+                     uint32_t BitWidth);
 
   /// Attempts to set the alignment of a pointer in the given address space.
   /// Returns an error description on failure.
@@ -223,7 +213,10 @@ public:
     DefaultGlobalsAddrSpace = DL.DefaultGlobalsAddrSpace;
     ManglingMode = DL.ManglingMode;
     LegalIntWidths = DL.LegalIntWidths;
-    Alignments = DL.Alignments;
+    IntAlignments = DL.IntAlignments;
+    FloatAlignments = DL.FloatAlignments;
+    VectorAlignments = DL.VectorAlignments;
+    StructAlignment = DL.StructAlignment;
     Pointers = DL.Pointers;
     NonIntegralAddressSpaces = DL.NonIntegralAddressSpaces;
     return *this;

diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 0208745..ae248e9 100644 (file)
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -102,23 +102,19 @@ unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const {
 // LayoutAlignElem, LayoutAlign support
 //===----------------------------------------------------------------------===//
 
-LayoutAlignElem LayoutAlignElem::get(AlignTypeEnum align_type, Align abi_align,
-                                     Align pref_align, uint32_t bit_width) {
-  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
+LayoutAlignElem LayoutAlignElem::get(Align ABIAlign, Align PrefAlign,
+                                     uint32_t BitWidth) {
+  assert(ABIAlign <= PrefAlign && "Preferred alignment worse than ABI!");
   LayoutAlignElem retval;
-  retval.AlignType = align_type;
-  retval.ABIAlign = abi_align;
-  retval.PrefAlign = pref_align;
-  retval.TypeBitWidth = bit_width;
+  retval.ABIAlign = ABIAlign;
+  retval.PrefAlign = PrefAlign;
+  retval.TypeBitWidth = BitWidth;
   return retval;
 }
 
-bool
-LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const {
-  return (AlignType == rhs.AlignType
-          && ABIAlign == rhs.ABIAlign
-          && PrefAlign == rhs.PrefAlign
-          && TypeBitWidth == rhs.TypeBitWidth);
+bool LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const {
+  return ABIAlign == rhs.ABIAlign && PrefAlign == rhs.PrefAlign &&
+         TypeBitWidth == rhs.TypeBitWidth;
 }
 
 //===----------------------------------------------------------------------===//
@@ -162,19 +158,18 @@ const char *DataLayout::getManglingComponent(const Triple &T) {
   return "-m:e";
 }
 
-static const LayoutAlignElem DefaultAlignments[] = {
-    {INTEGER_ALIGN, 1, Align(1), Align(1)},    // i1
-    {INTEGER_ALIGN, 8, Align(1), Align(1)},    // i8
-    {INTEGER_ALIGN, 16, Align(2), Align(2)},   // i16
-    {INTEGER_ALIGN, 32, Align(4), Align(4)},   // i32
-    {INTEGER_ALIGN, 64, Align(4), Align(8)},   // i64
-    {FLOAT_ALIGN, 16, Align(2), Align(2)},     // half, bfloat
-    {FLOAT_ALIGN, 32, Align(4), Align(4)},     // float
-    {FLOAT_ALIGN, 64, Align(8), Align(8)},     // double
-    {FLOAT_ALIGN, 128, Align(16), Align(16)},  // ppcf128, quad, ...
-    {VECTOR_ALIGN, 64, Align(8), Align(8)},    // v2i32, v1i64, ...
-    {VECTOR_ALIGN, 128, Align(16), Align(16)}, // v16i8, v8i16, v4i32, ...
-    {AGGREGATE_ALIGN, 0, Align(1), Align(8)}   // struct
+static const std::pair<AlignTypeEnum, LayoutAlignElem> DefaultAlignments[] = {
+    {INTEGER_ALIGN, {1, Align(1), Align(1)}},    // i1
+    {INTEGER_ALIGN, {8, Align(1), Align(1)}},    // i8
+    {INTEGER_ALIGN, {16, Align(2), Align(2)}},   // i16
+    {INTEGER_ALIGN, {32, Align(4), Align(4)}},   // i32
+    {INTEGER_ALIGN, {64, Align(4), Align(8)}},   // i64
+    {FLOAT_ALIGN, {16, Align(2), Align(2)}},     // half, bfloat
+    {FLOAT_ALIGN, {32, Align(4), Align(4)}},     // float
+    {FLOAT_ALIGN, {64, Align(8), Align(8)}},     // double
+    {FLOAT_ALIGN, {128, Align(16), Align(16)}},  // ppcf128, quad, ...
+    {VECTOR_ALIGN, {64, Align(8), Align(8)}},    // v2i32, v1i64, ...
+    {VECTOR_ALIGN, {128, Align(16), Align(16)}}, // v16i8, v8i16, v4i32, ...
 };
 
 void DataLayout::reset(StringRef Desc) {
@@ -190,11 +185,12 @@ void DataLayout::reset(StringRef Desc) {
   TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
   ManglingMode = MM_None;
   NonIntegralAddressSpaces.clear();
+  StructAlignment = LayoutAlignElem::get(Align(1), Align(8), 0);
 
   // Default alignments
-  for (const LayoutAlignElem &E : DefaultAlignments) {
-    if (Error Err = setAlignment((AlignTypeEnum)E.AlignType, E.ABIAlign,
-                                 E.PrefAlign, E.TypeBitWidth))
+  for (const auto &[Kind, Layout] : DefaultAlignments) {
+    if (Error Err = setAlignment(Kind, Layout.ABIAlign, Layout.PrefAlign,
+                                 Layout.TypeBitWidth))
       return report_fatal_error(std::move(Err));
   }
   if (Error Err = setPointerAlignmentInBits(0, Align(8), Align(8), 64, 64))
@@ -550,43 +546,63 @@ bool DataLayout::operator==(const DataLayout &Other) const {
              TheFunctionPtrAlignType == Other.TheFunctionPtrAlignType &&
              ManglingMode == Other.ManglingMode &&
              LegalIntWidths == Other.LegalIntWidths &&
-             Alignments == Other.Alignments && Pointers == Other.Pointers;
+             IntAlignments == Other.IntAlignments &&
+             FloatAlignments == Other.FloatAlignments &&
+             VectorAlignments == Other.VectorAlignments &&
+             StructAlignment == Other.StructAlignment &&
+             Pointers == Other.Pointers;
   // Note: getStringRepresentation() might differs, it is not canonicalized
   return Ret;
 }
 
-DataLayout::AlignmentsTy::iterator
-DataLayout::findAlignmentLowerBound(AlignTypeEnum AlignType,
-                                    uint32_t BitWidth) {
-  auto Pair = std::make_pair((unsigned)AlignType, BitWidth);
-  return partition_point(Alignments, [=](const LayoutAlignElem &E) {
-    return std::make_pair(E.AlignType, E.TypeBitWidth) < Pair;
+static SmallVectorImpl<LayoutAlignElem>::const_iterator
+findAlignmentLowerBound(const SmallVectorImpl<LayoutAlignElem> &Alignments,
+                        uint32_t BitWidth) {
+  return partition_point(Alignments, [BitWidth](const LayoutAlignElem &E) {
+    return E.TypeBitWidth < BitWidth;
   });
 }
 
-Error DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align,
-                               Align pref_align, uint32_t bit_width) {
+Error DataLayout::setAlignment(AlignTypeEnum AlignType, Align ABIAlign,
+                               Align PrefAlign, uint32_t BitWidth) {
   // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
   // uint16_t, it is unclear if there are requirements for alignment to be less
   // than 2^16 other than storage. In the meantime we leave the restriction as
   // an assert. See D67400 for context.
-  assert(Log2(abi_align) < 16 && Log2(pref_align) < 16 && "Alignment too big");
-  if (!isUInt<24>(bit_width))
+  assert(Log2(ABIAlign) < 16 && Log2(PrefAlign) < 16 && "Alignment too big");
+  if (!isUInt<24>(BitWidth))
     return reportError("Invalid bit width, must be a 24bit integer");
-  if (pref_align < abi_align)
+  if (PrefAlign < ABIAlign)
     return reportError(
         "Preferred alignment cannot be less than the ABI alignment");
 
-  AlignmentsTy::iterator I = findAlignmentLowerBound(align_type, bit_width);
-  if (I != Alignments.end() &&
-      I->AlignType == (unsigned)align_type && I->TypeBitWidth == bit_width) {
+  SmallVectorImpl<LayoutAlignElem> *Alignments;
+  switch (AlignType) {
+  case AGGREGATE_ALIGN:
+    StructAlignment.ABIAlign = ABIAlign;
+    StructAlignment.PrefAlign = PrefAlign;
+    return Error::success();
+  case INTEGER_ALIGN:
+    Alignments = &IntAlignments;
+    break;
+  case FLOAT_ALIGN:
+    Alignments = &FloatAlignments;
+    break;
+  case VECTOR_ALIGN:
+    Alignments = &VectorAlignments;
+    break;
+  }
+
+  auto I = partition_point(*Alignments, [BitWidth](const LayoutAlignElem &E) {
+    return E.TypeBitWidth < BitWidth;
+  });
+  if (I != Alignments->end() && I->TypeBitWidth == BitWidth) {
     // Update the abi, preferred alignments.
-    I->ABIAlign = abi_align;
-    I->PrefAlign = pref_align;
+    I->ABIAlign = ABIAlign;
+    I->PrefAlign = PrefAlign;
   } else {
     // Insert before I to keep the vector sorted.
-    Alignments.insert(I, LayoutAlignElem::get(align_type, abi_align,
-                                              pref_align, bit_width));
+    Alignments->insert(I, LayoutAlignElem::get(ABIAlign, PrefAlign, BitWidth));
   }
   return Error::success();
 }
@@ -633,13 +649,12 @@ Error DataLayout::setPointerAlignmentInBits(uint32_t AddrSpace, Align ABIAlign,
 
 Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
                                       bool abi_or_pref) const {
-  auto I = findAlignmentLowerBound(INTEGER_ALIGN, BitWidth);
+  auto I = findAlignmentLowerBound(IntAlignments, BitWidth);
   // If we don't have an exact match, use alignment of next larger integer
   // type. If there is none, use alignment of largest integer type by going
   // back one element.
-  if (I == Alignments.end() || I->AlignType != INTEGER_ALIGN)
+  if (I == IntAlignments.end())
     --I;
-  assert(I->AlignType == INTEGER_ALIGN && "Must be integer alignment");
   return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 }
 
@@ -668,7 +683,9 @@ public:
 
 void DataLayout::clear() {
   LegalIntWidths.clear();
-  Alignments.clear();
+  IntAlignments.clear();
+  FloatAlignments.clear();
+  VectorAlignments.clear();
   Pointers.clear();
   delete static_cast<StructLayoutMap *>(LayoutMap);
   LayoutMap = nullptr;
@@ -768,11 +785,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
 
     // Get the layout annotation... which is lazily created on demand.
     const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
-    const LayoutAlignElem &AggregateAlign = Alignments[0];
-    assert(AggregateAlign.AlignType == AGGREGATE_ALIGN &&
-           "Aggregate alignment must be first alignment entry");
     const Align Align =
-        abi_or_pref ? AggregateAlign.ABIAlign : AggregateAlign.PrefAlign;
+        abi_or_pref ? StructAlignment.ABIAlign : StructAlignment.PrefAlign;
     return std::max(Align, Layout->getAlignment());
   }
   case Type::IntegerTyID:
@@ -787,9 +801,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FP128TyID:
   case Type::X86_FP80TyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getFixedValue();
-    auto I = findAlignmentLowerBound(FLOAT_ALIGN, BitWidth);
-    if (I != Alignments.end() && I->AlignType == FLOAT_ALIGN &&
-        I->TypeBitWidth == BitWidth)
+    auto I = findAlignmentLowerBound(FloatAlignments, BitWidth);
+    if (I != FloatAlignments.end() && I->TypeBitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 
     // If we still couldn't find a reasonable default alignment, fall back
@@ -804,9 +817,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FixedVectorTyID:
   case Type::ScalableVectorTyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinValue();
-    auto I = findAlignmentLowerBound(VECTOR_ALIGN, BitWidth);
-    if (I != Alignments.end() && I->AlignType == VECTOR_ALIGN &&
-        I->TypeBitWidth == BitWidth)
+    auto I = findAlignmentLowerBound(VectorAlignments, BitWidth);
+    if (I != VectorAlignments.end() && I->TypeBitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 
     // By default, use natural alignment for vector types. This is consistent