Used when decomposing stores to skip the fields that are dying and the remainder if it is dying. Also allows forward sub to kick in for the inserted replacement locals.
phase.cpp
promotion.cpp
promotiondecomposition.cpp
+ promotionliveness.cpp
rangecheck.cpp
rationalize.cpp
redundantbranchopts.cpp
{
return lvTracked && lvType != TYP_STRUCT;
}
+#ifdef DEBUG
+ unsigned char lvTrackedWithoutIndex : 1; // Tracked but has no lvVarIndex (i.e. only valid GTF_VAR_DEATH flags, used
+ // by physical promotion)
+#endif
unsigned char lvPinned : 1; // is this a pinned variable?
unsigned char lvMustInit : 1; // must be initialized
printf(" ");
fieldVarDsc->PrintVarReg();
}
-
if (fieldVarDsc->lvTracked && fgLocalVarLivenessDone && tree->IsLastUse(index))
{
printf(" (last use)");
}
else // a normal not-promoted lclvar
{
- if (varDsc->lvTracked && fgLocalVarLivenessDone && ((tree->gtFlags & GTF_VAR_DEATH) != 0))
+ if ((varDsc->lvTracked || varDsc->lvTrackedWithoutIndex) && fgLocalVarLivenessDone &&
+ ((tree->gtFlags & GTF_VAR_DEATH) != 0))
{
printf(" (last use)");
}
// Start by assuming that the variable will be tracked.
varDsc->lvTracked = 1;
+ INDEBUG(varDsc->lvTrackedWithoutIndex = 0);
if (varDsc->lvRefCnt(lvaRefCountState) == 0)
{
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
#include "jitpch.h"
#include "promotion.h"
#include "jitstd/algorithm.h"
return a = (AccessKindFlags)((uint32_t)a & (uint32_t)b);
}
+//------------------------------------------------------------------------
+// OverlappingReplacements:
+// Find replacements that overlap the specified [offset..offset+size) interval.
+//
+// Parameters:
+// offset - Starting offset of interval
+// size - Size of interval
+// firstReplacement - [out] The first replacement that overlaps
+// endReplacement - [out, optional] One past the last replacement that overlaps
+//
+// Returns:
+// True if any replacement overlaps; otherwise false.
+//
+bool AggregateInfo::OverlappingReplacements(unsigned offset,
+ unsigned size,
+ Replacement** firstReplacement,
+ Replacement** endReplacement)
+{
+ size_t firstIndex = Promotion::BinarySearch<Replacement, &Replacement::Offset>(Replacements, offset);
+ if ((ssize_t)firstIndex < 0)
+ {
+ firstIndex = ~firstIndex;
+ if (firstIndex > 0)
+ {
+ Replacement& lastRepBefore = Replacements[firstIndex - 1];
+ if ((lastRepBefore.Offset + genTypeSize(lastRepBefore.AccessType)) > offset)
+ {
+ // Overlap with last entry starting before offs.
+ firstIndex--;
+ }
+ else if (firstIndex >= Replacements.size())
+ {
+ // Starts after last replacement ends.
+ return false;
+ }
+ }
+
+ const Replacement& first = Replacements[firstIndex];
+ if (first.Offset >= (offset + size))
+ {
+ // First candidate starts after this ends.
+ return false;
+ }
+ }
+
+ assert((firstIndex < Replacements.size()) && Replacements[firstIndex].Overlaps(offset, size));
+ *firstReplacement = &Replacements[firstIndex];
+
+ if (endReplacement != nullptr)
+ {
+ size_t lastIndex = Promotion::BinarySearch<Replacement, &Replacement::Offset>(Replacements, offset + size);
+ if ((ssize_t)lastIndex < 0)
+ {
+ lastIndex = ~lastIndex;
+ }
+
+ // Since we verified above that there is an overlapping replacement
+ // we know that lastIndex exists and is the next one that does not
+ // overlap.
+ assert(lastIndex > 0);
+ *endReplacement = Replacements.data() + lastIndex;
+ }
+
+ return true;
+}
+
// Tracks all the accesses into one particular struct local.
class LocalUses
{
// Parameters:
// comp - Compiler instance
// lclNum - Local num for this struct local
- // replacements - [out] Pointer to vector to create and insert replacements into
+ // aggregateInfo - [out] Pointer to aggregate info to create and insert replacements into.
//
- void PickPromotions(Compiler* comp, unsigned lclNum, jitstd::vector<Replacement>** replacements)
+ void PickPromotions(Compiler* comp, unsigned lclNum, AggregateInfo** aggregateInfo)
{
if (m_accesses.size() <= 0)
{
JITDUMP("Picking promotions for V%02u\n", lclNum);
- assert(*replacements == nullptr);
+ assert(*aggregateInfo == nullptr);
for (size_t i = 0; i < m_accesses.size(); i++)
{
const Access& access = m_accesses[i];
LclVarDsc* dsc = comp->lvaGetDesc(newLcl);
dsc->lvType = access.AccessType;
- if (*replacements == nullptr)
+ if (*aggregateInfo == nullptr)
{
- *replacements =
- new (comp, CMK_Promotion) jitstd::vector<Replacement>(comp->getAllocator(CMK_Promotion));
+ *aggregateInfo = new (comp, CMK_Promotion) AggregateInfo(comp->getAllocator(CMK_Promotion), lclNum);
}
- (*replacements)->push_back(Replacement(access.Offset, access.AccessType, newLcl DEBUGARG(bufp)));
+ (*aggregateInfo)
+ ->Replacements.push_back(Replacement(access.Offset, access.AccessType, newLcl DEBUGARG(bufp)));
}
JITDUMP("\n");
}
//------------------------------------------------------------------------
+// IntersectsOrAdjacent:
+// Check if this segment intersects or is adjacent to another segment.
+//
+// Parameters:
+// other - The other segment.
+//
+// Returns:
+// True if so.
+//
+bool StructSegments::Segment::IntersectsOrAdjacent(const Segment& other) const
+{
+ if (End < other.Start)
+ {
+ return false;
+ }
+
+ if (other.End < Start)
+ {
+ return false;
+ }
+
+ return true;
+}
+
+//------------------------------------------------------------------------
+// Contains:
+// Check if this segment contains another segment.
+//
+// Parameters:
+// other - The other segment.
+//
+// Returns:
+// True if so.
+//
+bool StructSegments::Segment::Contains(const Segment& other) const
+{
+ return (other.Start >= Start) && (other.End <= End);
+}
+
+//------------------------------------------------------------------------
+// Merge:
+// Update this segment to also contain another segment.
+//
+// Parameters:
+// other - The other segment.
+//
+void StructSegments::Segment::Merge(const Segment& other)
+{
+ Start = min(Start, other.Start);
+ End = max(End, other.End);
+}
+
+//------------------------------------------------------------------------
+// Add:
+// Add a segment to the data structure.
+//
+// Parameters:
+// segment - The segment to add.
+//
+void StructSegments::Add(const Segment& segment)
+{
+ size_t index = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.Start);
+
+ if ((ssize_t)index < 0)
+ {
+ index = ~index;
+ }
+
+ m_segments.insert(m_segments.begin() + index, segment);
+ size_t endIndex;
+ for (endIndex = index + 1; endIndex < m_segments.size(); endIndex++)
+ {
+ if (!m_segments[index].IntersectsOrAdjacent(m_segments[endIndex]))
+ {
+ break;
+ }
+
+ m_segments[index].Merge(m_segments[endIndex]);
+ }
+
+ m_segments.erase(m_segments.begin() + index + 1, m_segments.begin() + endIndex);
+}
+
+//------------------------------------------------------------------------
+// Subtract:
+// Subtract a segment from the data structure.
+//
+// Parameters:
+// segment - The segment to subtract.
+//
+void StructSegments::Subtract(const Segment& segment)
+{
+ size_t index = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.Start);
+ if ((ssize_t)index < 0)
+ {
+ index = ~index;
+ }
+ else
+ {
+ // Start == segment[index].End, which makes it non-interesting.
+ index++;
+ }
+
+ if (index >= m_segments.size())
+ {
+ return;
+ }
+
+ // Here we know Start < segment[index].End. Do they not intersect at all?
+ if (m_segments[index].Start >= segment.End)
+ {
+ // Does not intersect any segment.
+ return;
+ }
+
+ assert(m_segments[index].IntersectsOrAdjacent(segment));
+
+ if (m_segments[index].Contains(segment))
+ {
+ if (segment.Start > m_segments[index].Start)
+ {
+ // New segment (existing.Start, segment.Start)
+ if (segment.End < m_segments[index].End)
+ {
+ m_segments.insert(m_segments.begin() + index, Segment(m_segments[index].Start, segment.Start));
+
+ // And new segment (segment.End, existing.End)
+ m_segments[index + 1].Start = segment.End;
+ return;
+ }
+
+ m_segments[index].End = segment.Start;
+ return;
+ }
+ if (segment.End < m_segments[index].End)
+ {
+ // New segment (segment.End, existing.End)
+ m_segments[index].Start = segment.End;
+ return;
+ }
+
+ // Full segment is being removed
+ m_segments.erase(m_segments.begin() + index);
+ return;
+ }
+
+ if (segment.Start > m_segments[index].Start)
+ {
+ m_segments[index].End = segment.Start;
+ index++;
+ }
+
+ size_t endIndex = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.End);
+ if ((ssize_t)endIndex >= 0)
+ {
+ m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex + 1);
+ return;
+ }
+
+ endIndex = ~endIndex;
+ if (endIndex == m_segments.size())
+ {
+ m_segments.erase(m_segments.begin() + index, m_segments.end());
+ return;
+ }
+
+ if (segment.End > m_segments[endIndex].Start)
+ {
+ m_segments[endIndex].Start = segment.End;
+ }
+
+ m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex);
+}
+
+//------------------------------------------------------------------------
+// IsEmpty:
+// Check if the segment tree is empty.
+//
+// Returns:
+// True if so.
+//
+bool StructSegments::IsEmpty()
+{
+ return m_segments.size() == 0;
+}
+
+//------------------------------------------------------------------------
+// IsSingleSegment:
+// Check if the segment tree contains only a single segment, and return
+// it if so.
+//
+// Parameters:
+// result - [out] The single segment. Only valid if the method returns true.
+//
+// Returns:
+// True if so.
+//
+bool StructSegments::IsSingleSegment(Segment* result)
+{
+ if (m_segments.size() == 1)
+ {
+ *result = m_segments[0];
+ return true;
+ }
+
+ return false;
+}
+
+//------------------------------------------------------------------------
+// CoveringSegment:
+// Compute a segment that covers all contained segments in this segment tree.
+//
+// Parameters:
+// result - [out] The single segment. Only valid if the method returns true.
+//
+// Returns:
+// True if this segment tree was non-empty; otherwise false.
+//
+bool StructSegments::CoveringSegment(Segment* result)
+{
+ if (m_segments.size() == 0)
+ {
+ return false;
+ }
+
+ result->Start = m_segments[0].Start;
+ result->End = m_segments[m_segments.size() - 1].End;
+ return true;
+}
+
+#ifdef DEBUG
+//------------------------------------------------------------------------
+// Check:
+// Validate that the data structure is normalized and that it equals a
+// specific fixed bit vector.
+//
+// Parameters:
+// vect - The bit vector
+//
+// Remarks:
+// This validates that the internal representation is normalized (i.e.
+// all adjacent intervals are merged) and that it contains an index iff
+// the specified vector contains that index.
+//
+void StructSegments::Check(FixedBitVect* vect)
+{
+ bool first = true;
+ unsigned last = 0;
+ for (const Segment& segment : m_segments)
+ {
+ assert(first || (last < segment.Start));
+ assert(segment.End <= vect->bitVectGetSize());
+
+ for (unsigned i = last; i < segment.Start; i++)
+ assert(!vect->bitVectTest(i));
+
+ for (unsigned i = segment.Start; i < segment.End; i++)
+ assert(vect->bitVectTest(i));
+
+ first = false;
+ last = segment.End;
+ }
+
+ for (unsigned i = last, size = vect->bitVectGetSize(); i < size; i++)
+ assert(!vect->bitVectTest(i));
+}
+
+//------------------------------------------------------------------------
+// Dump:
+// Dump a string representation of the segment tree to stdout.
+//
+void StructSegments::Dump()
+{
+ if (m_segments.size() == 0)
+ {
+ printf("<empty>");
+ }
+ else
+ {
+ const char* sep = "";
+ for (const Segment& segment : m_segments)
+ {
+ printf("%s[%03u..%03u)", sep, segment.Start, segment.End);
+ sep = " ";
+ }
+ }
+}
+#endif
+
+//------------------------------------------------------------------------
+// SignificantSegments:
+// Compute a segment tree containing all significant (non-padding) segments
+// for the specified class layout.
+//
+// Parameters:
+// compiler - Compiler instance
+// layout - The layout
+// bitVectRept - In debug, a bit vector that represents the same segments as the returned segment tree.
+// Used for verification purposes.
+//
+// Returns:
+// Segment tree containing all significant parts of the layout.
+//
+StructSegments Promotion::SignificantSegments(Compiler* compiler,
+ ClassLayout* layout DEBUGARG(FixedBitVect** bitVectRepr))
+{
+ COMP_HANDLE compHnd = compiler->info.compCompHnd;
+
+ bool significantPadding;
+ if (layout->IsBlockLayout())
+ {
+ significantPadding = true;
+ JITDUMP(" Block op has significant padding due to block layout\n");
+ }
+ else
+ {
+ uint32_t attribs = compHnd->getClassAttribs(layout->GetClassHandle());
+ if ((attribs & CORINFO_FLG_INDEXABLE_FIELDS) != 0)
+ {
+ significantPadding = true;
+ JITDUMP(" Block op has significant padding due to indexable fields\n");
+ }
+ else if ((attribs & CORINFO_FLG_DONT_DIG_FIELDS) != 0)
+ {
+ significantPadding = true;
+ JITDUMP(" Block op has significant padding due to CORINFO_FLG_DONT_DIG_FIELDS\n");
+ }
+ else if (((attribs & CORINFO_FLG_CUSTOMLAYOUT) != 0) && ((attribs & CORINFO_FLG_CONTAINS_GC_PTR) == 0))
+ {
+ significantPadding = true;
+ JITDUMP(" Block op has significant padding due to CUSTOMLAYOUT without GC pointers\n");
+ }
+ else
+ {
+ significantPadding = false;
+ }
+ }
+
+ StructSegments segments(compiler->getAllocator(CMK_Promotion));
+
+ // Validate with "obviously correct" but less scalable fixed bit vector implementation.
+ INDEBUG(FixedBitVect* segmentBitVect = FixedBitVect::bitVectInit(layout->GetSize(), compiler));
+
+ if (significantPadding)
+ {
+ segments.Add(StructSegments::Segment(0, layout->GetSize()));
+
+#ifdef DEBUG
+ for (unsigned i = 0; i < layout->GetSize(); i++)
+ segmentBitVect->bitVectSet(i);
+#endif
+ }
+ else
+ {
+ unsigned numFields = compHnd->getClassNumInstanceFields(layout->GetClassHandle());
+ for (unsigned i = 0; i < numFields; i++)
+ {
+ CORINFO_FIELD_HANDLE fieldHnd = compHnd->getFieldInClass(layout->GetClassHandle(), (int)i);
+ unsigned fldOffset = compHnd->getFieldOffset(fieldHnd);
+ CORINFO_CLASS_HANDLE fieldClassHandle;
+ CorInfoType corType = compHnd->getFieldType(fieldHnd, &fieldClassHandle);
+ var_types varType = JITtype2varType(corType);
+ unsigned size = genTypeSize(varType);
+ if (size == 0)
+ {
+ // TODO-CQ: Recursively handle padding in sub structures
+ // here. Might be better to introduce a single JIT-EE call
+ // to query the significant segments -- that would also be
+ // usable by R2R even outside the version bubble in many
+ // cases.
+ size = compHnd->getClassSize(fieldClassHandle);
+ assert(size != 0);
+ }
+
+ segments.Add(StructSegments::Segment(fldOffset, fldOffset + size));
+#ifdef DEBUG
+ for (unsigned i = 0; i < size; i++)
+ segmentBitVect->bitVectSet(fldOffset + i);
+#endif
+ }
+ }
+
+#ifdef DEBUG
+ if (bitVectRepr != nullptr)
+ {
+ *bitVectRepr = segmentBitVect;
+ }
+#endif
+
+ // TODO-TP: Cache this per class layout, we call this for every struct
+ // operation on a promoted local.
+ return segments;
+}
+
+//------------------------------------------------------------------------
// CreateWriteBack:
// Create IR that writes a replacement local's value back to its struct local:
//
{
unsigned size = argNodeLcl->GetLayout(m_compiler)->GetSize();
WriteBackBefore(&arg.EarlyNodeRef(), argNodeLcl->GetLclNum(), argNodeLcl->GetLclOffs(), size);
+
+ if ((m_aggregates[argNodeLcl->GetLclNum()] != nullptr) && IsPromotedStructLocalDying(argNodeLcl))
+ {
+ argNodeLcl->gtFlags |= GTF_VAR_DEATH;
+ }
}
}
}
//------------------------------------------------------------------------
+// IsPromotedStructLocalDying:
+// Check if a promoted struct local is dying at its current position.
+//
+// Parameters:
+// lcl - The local
+//
+// Returns:
+// True if so.
+//
+// Remarks:
+// This effectively translates our precise liveness information for struct
+// uses into the liveness information that the rest of the JIT expects.
+//
+// If the remainder of the struct local is dying, then we expect that this
+// entire struct local is now dying, since all field accesses are going to be
+// replaced with other locals. The exception is if there is a queued read
+// back for any of the fields.
+//
+bool ReplaceVisitor::IsPromotedStructLocalDying(GenTreeLclVarCommon* lcl)
+{
+ StructDeaths deaths = m_liveness->GetDeathsForStructLocal(lcl);
+ if (!deaths.IsRemainderDying())
+ {
+ return false;
+ }
+
+ AggregateInfo* agg = m_aggregates[lcl->GetLclNum()];
+
+ for (size_t i = 0; i < agg->Replacements.size(); i++)
+ {
+ if (agg->Replacements[i].NeedsReadBack)
+ {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+//------------------------------------------------------------------------
// ReplaceLocal:
// Handle a local that may need to be replaced.
//
{
GenTreeLclVarCommon* lcl = (*use)->AsLclVarCommon();
unsigned lclNum = lcl->GetLclNum();
- if (m_replacements[lclNum] == nullptr)
+ if (m_aggregates[lclNum] == nullptr)
{
return;
}
- jitstd::vector<Replacement>& replacements = *m_replacements[lclNum];
+ jitstd::vector<Replacement>& replacements = m_aggregates[lclNum]->Replacements;
unsigned offs = lcl->GetLclOffs();
var_types accessType = lcl->TypeGet();
JITDUMP(" ..replaced with promoted lcl V%02u\n", rep.LclNum);
*use = m_compiler->gtNewLclvNode(rep.LclNum, accessType);
+ (*use)->gtFlags |= lcl->gtFlags & GTF_VAR_DEATH;
+
if ((lcl->gtFlags & GTF_VAR_DEF) != 0)
{
(*use)->gtFlags |= GTF_VAR_DEF; // TODO-ASG: delete.
//
void ReplaceVisitor::WriteBackBefore(GenTree** use, unsigned lcl, unsigned offs, unsigned size)
{
- if (m_replacements[lcl] == nullptr)
+ if (m_aggregates[lcl] == nullptr)
{
return;
}
- jitstd::vector<Replacement>& replacements = *m_replacements[lcl];
+ jitstd::vector<Replacement>& replacements = m_aggregates[lcl]->Replacements;
size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(replacements, offs);
if ((ssize_t)index < 0)
//
void ReplaceVisitor::MarkForReadBack(unsigned lcl, unsigned offs, unsigned size)
{
- if (m_replacements[lcl] == nullptr)
+ if (m_aggregates[lcl] == nullptr)
{
return;
}
- jitstd::vector<Replacement>& replacements = *m_replacements[lcl];
+ jitstd::vector<Replacement>& replacements = m_aggregates[lcl]->Replacements;
size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(replacements, offs);
if ((ssize_t)index < 0)
#endif
// Pick promotions based on the use information we just collected.
- bool anyReplacements = false;
- jitstd::vector<Replacement>** replacements =
- new (m_compiler, CMK_Promotion) jitstd::vector<Replacement>*[m_compiler->lvaCount]{};
+ bool anyReplacements = false;
+ jitstd::vector<AggregateInfo*> aggregates(m_compiler->lvaCount, nullptr, m_compiler->getAllocator(CMK_Promotion));
for (unsigned i = 0; i < numLocals; i++)
{
LocalUses* uses = localsUse.GetUsesByLocal(i);
continue;
}
- uses->PickPromotions(m_compiler, i, &replacements[i]);
+ uses->PickPromotions(m_compiler, i, &aggregates[i]);
- if (replacements[i] != nullptr)
+ if (aggregates[i] == nullptr)
{
- assert(replacements[i]->size() > 0);
- anyReplacements = true;
+ continue;
+ }
+
+ jitstd::vector<Replacement>& reps = aggregates[i]->Replacements;
+
+ assert(reps.size() > 0);
+ anyReplacements = true;
#ifdef DEBUG
- JITDUMP("V%02u promoted with %d replacements\n", i, (int)replacements[i]->size());
- for (const Replacement& rep : *replacements[i])
- {
- JITDUMP(" [%03u..%03u) promoted as %s V%02u\n", rep.Offset, rep.Offset + genTypeSize(rep.AccessType),
- varTypeName(rep.AccessType), rep.LclNum);
- }
+ JITDUMP("V%02u promoted with %d replacements\n", i, (int)reps.size());
+ for (const Replacement& rep : reps)
+ {
+ JITDUMP(" [%03u..%03u) promoted as %s V%02u\n", rep.Offset, rep.Offset + genTypeSize(rep.AccessType),
+ varTypeName(rep.AccessType), rep.LclNum);
+ }
#endif
+
+ JITDUMP("Computing unpromoted remainder for V%02u\n", i);
+ StructSegments unpromotedParts = SignificantSegments(m_compiler, m_compiler->lvaGetDesc(i)->GetLayout());
+ for (size_t i = 0; i < reps.size(); i++)
+ {
+ unpromotedParts.Subtract(
+ StructSegments::Segment(reps[i].Offset, reps[i].Offset + genTypeSize(reps[i].AccessType)));
+ }
+
+ JITDUMP(" Remainder: ");
+ DBEXEC(m_compiler->verbose, unpromotedParts.Dump());
+ JITDUMP("\n\n");
+
+ StructSegments::Segment unpromotedSegment;
+ if (unpromotedParts.CoveringSegment(&unpromotedSegment))
+ {
+ aggregates[i]->UnpromotedMin = unpromotedSegment.Start;
+ aggregates[i]->UnpromotedMax = unpromotedSegment.End;
+ assert(unpromotedSegment.Start < unpromotedSegment.End);
+ }
+ else
+ {
+ // Aggregate is fully promoted, leave UnpromotedMin == UnpromotedMax to indicate this.
}
}
return PhaseStatus::MODIFIED_NOTHING;
}
+ // Compute liveness for the fields and remainders.
+ PromotionLiveness liveness(m_compiler, aggregates);
+ liveness.Run();
+
+ JITDUMP("Making replacements\n\n");
// Make all replacements we decided on.
- ReplaceVisitor replacer(this, replacements);
+ ReplaceVisitor replacer(this, aggregates, &liveness);
for (BasicBlock* bb : m_compiler->Blocks())
{
for (Statement* stmt : bb->Statements())
for (unsigned i = 0; i < numLocals; i++)
{
- if (replacements[i] == nullptr)
+ if (aggregates[i] == nullptr)
{
continue;
}
- for (Replacement& rep : *replacements[i])
+ for (size_t j = 0; j < aggregates[i]->Replacements.size(); j++)
{
+ Replacement& rep = aggregates[i]->Replacements[j];
assert(!rep.NeedsReadBack || !rep.NeedsWriteBack);
if (rep.NeedsReadBack)
{
- JITDUMP("Reading back replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB ":\n", i,
- rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum);
+ if (liveness.IsReplacementLiveOut(bb, i, (unsigned)j))
+ {
+ JITDUMP("Reading back replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB ":\n", i,
+ rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum);
- GenTree* readBack = CreateReadBack(m_compiler, i, rep);
- Statement* stmt = m_compiler->fgNewStmtFromTree(readBack);
- DISPSTMT(stmt);
- m_compiler->fgInsertStmtNearEnd(bb, stmt);
+ GenTree* readBack = CreateReadBack(m_compiler, i, rep);
+ Statement* stmt = m_compiler->fgNewStmtFromTree(readBack);
+ DISPSTMT(stmt);
+ m_compiler->fgInsertStmtNearEnd(bb, stmt);
+ }
+ else
+ {
+ JITDUMP(
+ "Skipping reading back dead replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB
+ "\n",
+ i, rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum);
+ }
rep.NeedsReadBack = false;
}
Statement* prevStmt = nullptr;
for (unsigned lclNum = 0; lclNum < numLocals; lclNum++)
{
- if (replacements[lclNum] == nullptr)
+ if (aggregates[lclNum] == nullptr)
{
continue;
}
LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum);
if (dsc->lvIsParam || dsc->lvIsOSRLocal)
{
- InsertInitialReadBack(lclNum, *replacements[lclNum], &prevStmt);
+ InsertInitialReadBack(lclNum, aggregates[lclNum]->Replacements, &prevStmt);
}
else if (dsc->lvSuppressedZeroInit)
{
// Now that we are promoting some fields that assumption may be
// invalidated for those fields, and we may need to insert explicit
// zero inits again.
- ExplicitlyZeroInitReplacementLocals(lclNum, *replacements[lclNum], &prevStmt);
+ ExplicitlyZeroInitReplacementLocals(lclNum, aggregates[lclNum]->Replacements, &prevStmt);
}
}
bool Overlaps(unsigned otherStart, unsigned otherSize) const;
};
+// Represents significant segments of a struct operation.
+//
+// Essentially a segment tree (but not stored as a tree) that supports boolean
+// Add/Subtract operations of segments. Used to compute the remainder after
+// replacements have been handled as part of a decomposed block operation.
+class StructSegments
+{
+public:
+ struct Segment
+ {
+ unsigned Start = 0;
+ unsigned End = 0;
+
+ Segment()
+ {
+ }
+
+ Segment(unsigned start, unsigned end) : Start(start), End(end)
+ {
+ }
+
+ bool IntersectsOrAdjacent(const Segment& other) const;
+ bool Contains(const Segment& other) const;
+ void Merge(const Segment& other);
+ };
+
+private:
+ jitstd::vector<Segment> m_segments;
+
+public:
+ StructSegments(CompAllocator allocator) : m_segments(allocator)
+ {
+ }
+
+ void Add(const Segment& segment);
+ void Subtract(const Segment& segment);
+ bool IsEmpty();
+ bool IsSingleSegment(Segment* result);
+ bool CoveringSegment(Segment* result);
+
+#ifdef DEBUG
+ void Check(FixedBitVect* vect);
+ void Dump();
+#endif
+};
+
+// Represents information about an aggregate that now has replacements in it.
+struct AggregateInfo
+{
+ jitstd::vector<Replacement> Replacements;
+ unsigned LclNum;
+ // Min offset in the struct local of the unpromoted part.
+ unsigned UnpromotedMin = 0;
+ // Max offset in the struct local of the unpromoted part.
+ unsigned UnpromotedMax = 0;
+
+ AggregateInfo(CompAllocator alloc, unsigned lclNum) : Replacements(alloc), LclNum(lclNum)
+ {
+ }
+
+ bool OverlappingReplacements(unsigned offset,
+ unsigned size,
+ Replacement** firstReplacement,
+ Replacement** endReplacement);
+};
+
class Promotion
{
Compiler* m_compiler;
friend class LocalUses;
friend class LocalsUseVisitor;
+ friend struct AggregateInfo;
+ friend class PromotionLiveness;
friend class ReplaceVisitor;
friend class DecompositionPlan;
friend class StructSegments;
+ static StructSegments SignificantSegments(Compiler* compiler,
+ ClassLayout* layout DEBUGARG(FixedBitVect** bitVectRepr = nullptr));
+
void InsertInitialReadBack(unsigned lclNum, const jitstd::vector<Replacement>& replacements, Statement** prevStmt);
void ExplicitlyZeroInitReplacementLocals(unsigned lclNum,
const jitstd::vector<Replacement>& replacements,
PhaseStatus Run();
};
+// Class to represent liveness information for a struct local's fields and remainder.
+class StructDeaths
+{
+ BitVec m_deaths;
+ unsigned m_numFields = 0;
+
+ friend class PromotionLiveness;
+
+private:
+ StructDeaths(BitVec deaths, unsigned numFields) : m_deaths(deaths), m_numFields(numFields)
+ {
+ }
+
+public:
+ StructDeaths() : m_deaths(BitVecOps::UninitVal())
+ {
+ }
+
+ bool IsRemainderDying() const;
+ bool IsReplacementDying(unsigned index) const;
+};
+
+struct BasicBlockLiveness;
+
+// Class to compute and track liveness information pertaining promoted structs.
+class PromotionLiveness
+{
+ Compiler* m_compiler;
+ jitstd::vector<AggregateInfo*>& m_aggregates;
+ BitVecTraits* m_bvTraits = nullptr;
+ unsigned* m_structLclToTrackedIndex = nullptr;
+ unsigned m_numVars = 0;
+ BasicBlockLiveness* m_bbInfo = nullptr;
+ bool m_hasPossibleBackEdge = false;
+ BitVec m_liveIn;
+ BitVec m_ehLiveVars;
+ JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, BitVec> m_aggDeaths;
+
+public:
+ PromotionLiveness(Compiler* compiler, jitstd::vector<AggregateInfo*>& aggregates)
+ : m_compiler(compiler), m_aggregates(aggregates), m_aggDeaths(compiler->getAllocator(CMK_Promotion))
+ {
+ }
+
+ void Run();
+ bool IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacement);
+ StructDeaths GetDeathsForStructLocal(GenTreeLclVarCommon* use);
+
+private:
+ void MarkUseDef(GenTreeLclVarCommon* lcl, BitVec& useSet, BitVec& defSet);
+ void MarkIndex(unsigned index, bool isUse, bool isDef, BitVec& useSet, BitVec& defSet);
+ void ComputeUseDefSets();
+ void InterBlockLiveness();
+ bool PerBlockLiveness(BasicBlock* block);
+ void AddHandlerLiveVars(BasicBlock* block, BitVec& ehLiveVars);
+ void FillInLiveness();
+ void FillInLiveness(BitVec& life, BitVec volatileVars, GenTreeLclVarCommon* lcl);
+#ifdef DEBUG
+ void DumpVarSet(BitVec set, BitVec allVars);
+#endif
+};
+
class DecompositionStatementList;
class DecompositionPlan;
class ReplaceVisitor : public GenTreeVisitor<ReplaceVisitor>
{
- Promotion* m_prom;
- jitstd::vector<Replacement>** m_replacements;
- bool m_madeChanges = false;
+ Promotion* m_prom;
+ jitstd::vector<AggregateInfo*>& m_aggregates;
+ PromotionLiveness* m_liveness;
+ bool m_madeChanges = false;
public:
enum
UseExecutionOrder = true,
};
- ReplaceVisitor(Promotion* prom, jitstd::vector<Replacement>** replacements)
- : GenTreeVisitor(prom->m_compiler), m_prom(prom), m_replacements(replacements)
+ ReplaceVisitor(Promotion* prom, jitstd::vector<AggregateInfo*>& aggregates, PromotionLiveness* liveness)
+ : GenTreeVisitor(prom->m_compiler), m_prom(prom), m_aggregates(aggregates), m_liveness(liveness)
{
}
private:
void LoadStoreAroundCall(GenTreeCall* call, GenTree* user);
+ bool IsPromotedStructLocalDying(GenTreeLclVarCommon* structLcl);
void ReplaceLocal(GenTree** use, GenTree* user);
void StoreBeforeReturn(GenTreeUnOp* ret);
void WriteBackBefore(GenTree** use, unsigned lcl, unsigned offs, unsigned size);
Replacement* srcEndRep,
DecompositionStatementList* statements,
DecompositionPlan* plan);
+#ifdef DEBUG
+ const char* LastUseString(GenTreeLclVarCommon* lcl, Replacement* rep);
+#endif
};
#endif
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
#include "jitpch.h"
#include "promotion.h"
#include "jitstd/algorithm.h"
}
};
-// Represents significant segments of a struct operation.
-//
-// Essentially a segment tree (but not stored as a tree) that supports boolean
-// Add/Subtract operations of segments. Used to compute the remainder after
-// replacements have been handled as part of a decomposed block operation.
-class StructSegments
-{
-public:
- struct Segment
- {
- unsigned Start = 0;
- unsigned End = 0;
-
- Segment()
- {
- }
-
- Segment(unsigned start, unsigned end) : Start(start), End(end)
- {
- }
-
- bool IntersectsInclusive(const Segment& other) const
- {
- if (End < other.Start)
- {
- return false;
- }
-
- if (other.End < Start)
- {
- return false;
- }
-
- return true;
- }
-
- bool Contains(const Segment& other) const
- {
- return other.Start >= Start && other.End <= End;
- }
-
- void Merge(const Segment& other)
- {
- Start = min(Start, other.Start);
- End = max(End, other.End);
- }
- };
-
-private:
- jitstd::vector<Segment> m_segments;
-
-public:
- StructSegments(CompAllocator allocator) : m_segments(allocator)
- {
- }
-
- //------------------------------------------------------------------------
- // Add:
- // Add a segment to the data structure.
- //
- // Parameters:
- // segment - The segment to add.
- //
- void Add(const Segment& segment)
- {
- size_t index = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.Start);
-
- if ((ssize_t)index < 0)
- {
- index = ~index;
- }
-
- m_segments.insert(m_segments.begin() + index, segment);
- size_t endIndex;
- for (endIndex = index + 1; endIndex < m_segments.size(); endIndex++)
- {
- if (!m_segments[index].IntersectsInclusive(m_segments[endIndex]))
- {
- break;
- }
-
- m_segments[index].Merge(m_segments[endIndex]);
- }
-
- m_segments.erase(m_segments.begin() + index + 1, m_segments.begin() + endIndex);
- }
-
- //------------------------------------------------------------------------
- // Subtract:
- // Subtract a segment from the data structure.
- //
- // Parameters:
- // segment - The segment to subtract.
- //
- void Subtract(const Segment& segment)
- {
- size_t index = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.Start);
- if ((ssize_t)index < 0)
- {
- index = ~index;
- }
- else
- {
- // Start == segment[index].End, which makes it non-interesting.
- index++;
- }
-
- if (index >= m_segments.size())
- {
- return;
- }
-
- // Here we know Start < segment[index].End. Do they not intersect at all?
- if (m_segments[index].Start >= segment.End)
- {
- // Does not intersect any segment.
- return;
- }
-
- assert(m_segments[index].IntersectsInclusive(segment));
-
- if (m_segments[index].Contains(segment))
- {
- if (segment.Start > m_segments[index].Start)
- {
- // New segment (existing.Start, segment.Start)
- if (segment.End < m_segments[index].End)
- {
- m_segments.insert(m_segments.begin() + index, Segment(m_segments[index].Start, segment.Start));
-
- // And new segment (segment.End, existing.End)
- m_segments[index + 1].Start = segment.End;
- return;
- }
-
- m_segments[index].End = segment.Start;
- return;
- }
- if (segment.End < m_segments[index].End)
- {
- // New segment (segment.End, existing.End)
- m_segments[index].Start = segment.End;
- return;
- }
-
- // Full segment is being removed
- m_segments.erase(m_segments.begin() + index);
- return;
- }
-
- if (segment.Start > m_segments[index].Start)
- {
- m_segments[index].End = segment.Start;
- index++;
- }
-
- size_t endIndex = Promotion::BinarySearch<Segment, &Segment::End>(m_segments, segment.End);
- if ((ssize_t)endIndex >= 0)
- {
- m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex + 1);
- return;
- }
-
- endIndex = ~endIndex;
- if (endIndex == m_segments.size())
- {
- m_segments.erase(m_segments.begin() + index, m_segments.end());
- return;
- }
-
- if (segment.End > m_segments[endIndex].Start)
- {
- m_segments[endIndex].Start = segment.End;
- }
-
- m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex);
- }
-
- //------------------------------------------------------------------------
- // IsEmpty:
- // Check if the segment tree is empty.
- //
- // Returns:
- // True if so.
- //
- bool IsEmpty()
- {
- return m_segments.size() == 0;
- }
-
- //------------------------------------------------------------------------
- // IsSingleSegment:
- // Check if the segment tree contains only a single segment, and return
- // it if so.
- //
- // Parameters:
- // result - [out] The single segment. Only valid if the method returns true.
- //
- // Returns:
- // True if so.
- //
- bool IsSingleSegment(Segment* result)
- {
- if (m_segments.size() == 1)
- {
- *result = m_segments[0];
- return true;
- }
-
- return false;
- }
-
-#ifdef DEBUG
- //------------------------------------------------------------------------
- // Check:
- // Validate that the data structure is normalized and that it equals a
- // specific fixed bit vector.
- //
- // Parameters:
- // vect - The bit vector
- //
- // Remarks:
- // This validates that the internal representation is normalized (i.e.
- // all adjacent intervals are merged) and that it contains an index iff
- // the specified vector contains that index.
- //
- void Check(FixedBitVect* vect)
- {
- bool first = true;
- unsigned last = 0;
- for (const Segment& segment : m_segments)
- {
- assert(first || (last < segment.Start));
- assert(segment.End <= vect->bitVectGetSize());
-
- for (unsigned i = last; i < segment.Start; i++)
- assert(!vect->bitVectTest(i));
-
- for (unsigned i = segment.Start; i < segment.End; i++)
- assert(vect->bitVectTest(i));
-
- first = false;
- last = segment.End;
- }
-
- for (unsigned i = last, size = vect->bitVectGetSize(); i < size; i++)
- assert(!vect->bitVectTest(i));
- }
-
- //------------------------------------------------------------------------
- // Dump:
- // Dump a string representation of the segment tree to stdout.
- //
- void Dump()
- {
- if (m_segments.size() == 0)
- {
- printf("<empty>");
- }
- else
- {
- const char* sep = "";
- for (const Segment& segment : m_segments)
- {
- printf("%s[%03u..%03u)", sep, segment.Start, segment.End);
- sep = " ";
- }
- }
- }
-#endif
-};
-
// Represents a plan for decomposing a block operation into direct treatment of
// replacement fields and the remainder.
class DecompositionPlan
var_types Type;
};
- Compiler* m_compiler;
- ArrayStack<Entry> m_entries;
- GenTree* m_dst;
- GenTree* m_src;
- bool m_srcInvolvesReplacements;
+ Compiler* m_compiler;
+ jitstd::vector<AggregateInfo*>& m_aggregates;
+ PromotionLiveness* m_liveness;
+ GenTree* m_dst;
+ GenTree* m_src;
+ bool m_dstInvolvesReplacements;
+ bool m_srcInvolvesReplacements;
+ ArrayStack<Entry> m_entries;
+ bool m_hasNonRemainderUseOfStructLocal = false;
public:
- DecompositionPlan(Compiler* comp, GenTree* dst, GenTree* src, bool srcInvolvesReplacements)
+ DecompositionPlan(Compiler* comp,
+ jitstd::vector<AggregateInfo*>& aggregates,
+ PromotionLiveness* liveness,
+ GenTree* dst,
+ GenTree* src,
+ bool dstInvolvesReplacements,
+ bool srcInvolvesReplacements)
: m_compiler(comp)
- , m_entries(comp->getAllocator(CMK_Promotion))
+ , m_aggregates(aggregates)
+ , m_liveness(liveness)
, m_dst(dst)
, m_src(src)
+ , m_dstInvolvesReplacements(dstInvolvesReplacements)
, m_srcInvolvesReplacements(srcInvolvesReplacements)
+ , m_entries(comp->getAllocator(CMK_Promotion))
{
}
}
//------------------------------------------------------------------------
+ // MarkNonRemainderUseOfStructLocal:
+ // Mark that some of the destination replacements are being handled via a
+ // readback. This invalidates liveness information for the remainder
+ // because the struct local will now also be used for the readback.
+ //
+ void MarkNonRemainderUseOfStructLocal()
+ {
+ m_hasNonRemainderUseOfStructLocal = true;
+ }
+
+ //------------------------------------------------------------------------
// Finalize:
// Create IR to perform the full decomposed struct copy as specified by
// the entries that were added to the decomposition plan. Add the
{
ClassLayout* dstLayout = m_dst->GetLayout(m_compiler);
- COMP_HANDLE compHnd = m_compiler->info.compCompHnd;
-
- bool significantPadding;
- if (dstLayout->IsBlockLayout())
- {
- significantPadding = true;
- JITDUMP(" Block op has significant padding due to block layout\n");
- }
- else
- {
- uint32_t attribs = compHnd->getClassAttribs(dstLayout->GetClassHandle());
- if ((attribs & CORINFO_FLG_INDEXABLE_FIELDS) != 0)
- {
- significantPadding = true;
- JITDUMP(" Block op has significant padding due to indexable fields\n");
- }
- else if ((attribs & CORINFO_FLG_DONT_DIG_FIELDS) != 0)
- {
- significantPadding = true;
- JITDUMP(" Block op has significant padding due to CORINFO_FLG_DONT_DIG_FIELDS\n");
- }
- else if (((attribs & CORINFO_FLG_CUSTOMLAYOUT) != 0) && ((attribs & CORINFO_FLG_CONTAINS_GC_PTR) == 0))
- {
- significantPadding = true;
- JITDUMP(" Block op has significant padding due to CUSTOMLAYOUT without GC pointers\n");
- }
- else
- {
- significantPadding = false;
- }
- }
-
- StructSegments segments(m_compiler->getAllocator(CMK_Promotion));
-
// Validate with "obviously correct" but less scalable fixed bit vector implementation.
- INDEBUG(FixedBitVect* segmentBitVect = FixedBitVect::bitVectInit(dstLayout->GetSize(), m_compiler));
-
- if (significantPadding)
- {
- segments.Add(StructSegments::Segment(0, dstLayout->GetSize()));
-
-#ifdef DEBUG
- for (unsigned i = 0; i < dstLayout->GetSize(); i++)
- segmentBitVect->bitVectSet(i);
-#endif
- }
- else
- {
- unsigned numFields = compHnd->getClassNumInstanceFields(dstLayout->GetClassHandle());
- for (unsigned i = 0; i < numFields; i++)
- {
- CORINFO_FIELD_HANDLE fieldHnd = compHnd->getFieldInClass(dstLayout->GetClassHandle(), (int)i);
- unsigned fldOffset = compHnd->getFieldOffset(fieldHnd);
- CORINFO_CLASS_HANDLE fieldClassHandle;
- CorInfoType corType = compHnd->getFieldType(fieldHnd, &fieldClassHandle);
- var_types varType = JITtype2varType(corType);
- unsigned size = genTypeSize(varType);
- if (size == 0)
- {
- // TODO-CQ: Recursively handle padding in sub structures
- // here. Might be better to introduce a single JIT-EE call
- // to query the significant segments -- that would also be
- // usable by R2R even outside the version bubble in many
- // cases.
- size = compHnd->getClassSize(fieldClassHandle);
- assert(size != 0);
- }
-
- segments.Add(StructSegments::Segment(fldOffset, fldOffset + size));
-#ifdef DEBUG
- for (unsigned i = 0; i < size; i++)
- segmentBitVect->bitVectSet(fldOffset + i);
-#endif
- }
- }
-
- // TODO-TP: Cache above StructSegments per class layout and just clone
- // it there before the following subtract operations.
+ INDEBUG(FixedBitVect * segmentBitVect);
+ StructSegments segments = Promotion::SignificantSegments(m_compiler, dstLayout DEBUGARG(&segmentBitVect));
for (int i = 0; i < m_entries.Height(); i++)
{
// the rest of the remainder); or by handling a specific 'hole' as a
// primitive.
//
- RemainderStrategy DetermineRemainderStrategy()
+ RemainderStrategy DetermineRemainderStrategy(const StructDeaths& dstDeaths)
{
+ if (m_dstInvolvesReplacements && !m_hasNonRemainderUseOfStructLocal && dstDeaths.IsRemainderDying())
+ {
+ JITDUMP(" => Remainder strategy: do nothing (remainder dying)\n");
+ return RemainderStrategy(RemainderStrategy::NoRemainder);
+ }
+
StructSegments remainder = ComputeRemainder();
if (remainder.IsEmpty())
{
- JITDUMP(" => Remainder strategy: do nothing\n");
+ JITDUMP(" => Remainder strategy: do nothing (no remainder)\n");
return RemainderStrategy(RemainderStrategy::NoRemainder);
}
{
var_types primitiveType = TYP_UNDEF;
unsigned size = segment.End - segment.Start;
- // For
if ((size == TARGET_POINTER_SIZE) && ((segment.Start % TARGET_POINTER_SIZE) == 0))
{
ClassLayout* dstLayout = m_dst->GetLayout(m_compiler);
{
if (!IsInit() || CanInitPrimitive(primitiveType))
{
- JITDUMP(" => Remainder strategy: %s at %03u\n", varTypeName(primitiveType), segment.Start);
+ JITDUMP(" => Remainder strategy: %s at +%03u\n", varTypeName(primitiveType), segment.Start);
return RemainderStrategy(RemainderStrategy::Primitive, segment.Start, primitiveType);
}
else
//
void FinalizeInit(DecompositionStatementList* statements)
{
- uint8_t initPattern = GetInitPattern();
+ uint8_t initPattern = GetInitPattern();
+ StructDeaths deaths = m_liveness->GetDeathsForStructLocal(m_dst->AsLclVarCommon());
+
+ AggregateInfo* agg = m_aggregates[m_dst->AsLclVarCommon()->GetLclNum()];
+ assert((agg != nullptr) && (agg->Replacements.size() > 0));
+ Replacement* firstRep = agg->Replacements.data();
for (int i = 0; i < m_entries.Height(); i++)
{
const Entry& entry = m_entries.BottomRef(i);
assert((entry.ToLclNum != BAD_VAR_NUM) && (entry.ToReplacement != nullptr));
- GenTree* src = m_compiler->gtNewConWithPattern(entry.Type, initPattern);
- GenTree* dst = m_compiler->gtNewLclvNode(entry.ToLclNum, entry.Type);
- statements->AddStatement(m_compiler->gtNewAssignNode(dst, src));
+ assert((entry.ToReplacement >= firstRep) && (entry.ToReplacement < firstRep + agg->Replacements.size()));
+ size_t replacementIndex = entry.ToReplacement - firstRep;
+
+ if (!deaths.IsReplacementDying((unsigned)replacementIndex))
+ {
+ GenTree* src = m_compiler->gtNewConWithPattern(entry.Type, initPattern);
+ GenTree* dst = m_compiler->gtNewLclvNode(entry.ToLclNum, entry.Type);
+ statements->AddStatement(m_compiler->gtNewAssignNode(dst, src));
+ }
+
entry.ToReplacement->NeedsWriteBack = true;
entry.ToReplacement->NeedsReadBack = false;
}
- RemainderStrategy remainderStrategy = DetermineRemainderStrategy();
+ RemainderStrategy remainderStrategy = DetermineRemainderStrategy(deaths);
if (remainderStrategy.Type == RemainderStrategy::FullBlock)
{
GenTree* asg = m_compiler->gtNewAssignNode(m_dst, m_src);
{
assert(m_dst->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_BLK) && m_src->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_BLK));
- RemainderStrategy remainderStrategy = DetermineRemainderStrategy();
+ StructDeaths dstDeaths;
+ if (m_dstInvolvesReplacements)
+ {
+ dstDeaths = m_liveness->GetDeathsForStructLocal(m_dst->AsLclVarCommon());
+ }
+
+ RemainderStrategy remainderStrategy = DetermineRemainderStrategy(dstDeaths);
// If the remainder is a full block and is going to incur write barrier
// then avoid incurring multiple write barriers for each source
{
for (int i = 0; i < m_entries.Height(); i++)
{
- if (!IsHandledByRemainder(m_entries.BottomRef(i), remainderStrategy))
+ if (!CanSkipEntry(m_entries.BottomRef(i), dstDeaths, remainderStrategy))
{
numAddrUses++;
}
// See if our first indirection will subsume the null check (usual case).
for (int i = 0; i < m_entries.Height(); i++)
{
- if (IsHandledByRemainder(m_entries.BottomRef(i), remainderStrategy))
+ if (CanSkipEntry(m_entries.BottomRef(i), dstDeaths, remainderStrategy))
{
continue;
}
statements->AddStatement(indir);
}
+ StructDeaths srcDeaths;
+ if (m_srcInvolvesReplacements)
+ {
+ srcDeaths = m_liveness->GetDeathsForStructLocal(m_src->AsLclVarCommon());
+ }
+
for (int i = 0; i < m_entries.Height(); i++)
{
const Entry& entry = m_entries.BottomRef(i);
- if (IsHandledByRemainder(entry, remainderStrategy))
+ if (entry.ToReplacement != nullptr)
{
- assert(entry.FromReplacement != nullptr);
- JITDUMP(" Skipping dst+%03u <- V%02u (%s); it is up-to-date in its struct local and will be handled "
+ entry.ToReplacement->NeedsWriteBack = true;
+ entry.ToReplacement->NeedsReadBack = false;
+ }
+
+ if (CanSkipEntry(entry, dstDeaths, remainderStrategy))
+ {
+ if (entry.FromReplacement != nullptr)
+ {
+ JITDUMP(
+ " Skipping dst+%03u <- V%02u (%s); it is up-to-date in its struct local and will be handled "
"as part of the remainder\n",
entry.Offset, entry.FromReplacement->LclNum, entry.FromReplacement->Description);
+ }
+ else if (entry.ToReplacement != nullptr)
+ {
+ JITDUMP(" Skipping def of V%02u (%s); it is dying", entry.ToReplacement->LclNum,
+ entry.ToReplacement->Description);
+ }
+
continue;
}
if (entry.FromLclNum != BAD_VAR_NUM)
{
src = m_compiler->gtNewLclvNode(entry.FromLclNum, entry.Type);
+
+ if (entry.FromReplacement != nullptr)
+ {
+ AggregateInfo* srcAgg = m_aggregates[m_src->AsLclVarCommon()->GetLclNum()];
+ Replacement* firstRep = srcAgg->Replacements.data();
+ assert((entry.FromReplacement >= firstRep) &&
+ (entry.FromReplacement < (firstRep + srcAgg->Replacements.size())));
+ size_t replacementIndex = entry.FromReplacement - firstRep;
+ if (srcDeaths.IsReplacementDying((unsigned)replacementIndex))
+ {
+ src->gtFlags |= GTF_VAR_DEATH;
+ }
+ }
}
else if (m_src->OperIs(GT_LCL_VAR, GT_LCL_FLD))
{
}
statements->AddStatement(m_compiler->gtNewAssignNode(dst, src));
- if (entry.ToReplacement != nullptr)
- {
- entry.ToReplacement->NeedsWriteBack = true;
- entry.ToReplacement->NeedsReadBack = false;
- }
}
if ((remainderStrategy.Type == RemainderStrategy::FullBlock) && !m_srcInvolvesReplacements)
}
//------------------------------------------------------------------------
- // IsHandledByRemainder:
- // Check if the specified entry is redundant because the remainder would
- // handle it anyway. This occurs when we have a source replacement that
- // is up-to-date in its struct local and we are going to retain a full
- // block operation anyway.
+ // CanSkipEntry:
+ // Check if the specified entry can be skipped because it is writing to a
+ // death replacement or because the remainder would handle it anyway.
//
// Parameters:
// entry - The init/copy entry
// remainderStrategy - The strategy we are using for the remainder
//
- bool IsHandledByRemainder(const Entry& entry, const RemainderStrategy& remainderStrategy)
+ bool CanSkipEntry(const Entry& entry, const StructDeaths& deaths, const RemainderStrategy& remainderStrategy)
{
- return (remainderStrategy.Type == RemainderStrategy::FullBlock) && (entry.FromReplacement != nullptr) &&
- !entry.FromReplacement->NeedsWriteBack && (entry.ToLclNum == BAD_VAR_NUM);
+ if (entry.ToReplacement != nullptr)
+ {
+ // Check if this entry is dying anyway.
+ assert(m_dstInvolvesReplacements);
+
+ AggregateInfo* agg = m_aggregates[m_dst->AsLclVarCommon()->GetLclNum()];
+ assert((agg != nullptr) && (agg->Replacements.size() > 0));
+ Replacement* firstRep = agg->Replacements.data();
+ assert((entry.ToReplacement >= firstRep) && (entry.ToReplacement < (firstRep + agg->Replacements.size())));
+
+ size_t replacementIndex = entry.ToReplacement - firstRep;
+ if (deaths.IsReplacementDying((unsigned)replacementIndex))
+ {
+ return true;
+ }
+ }
+
+ if (entry.FromReplacement != nullptr)
+ {
+ // Check if the remainder is going to handle it.
+ return (remainderStrategy.Type == RemainderStrategy::FullBlock) && !entry.FromReplacement->NeedsWriteBack &&
+ (entry.ToLclNum == BAD_VAR_NUM);
+ }
+
+ return false;
}
//------------------------------------------------------------------------
if (!dstInvolvesReplacements && !srcInvolvesReplacements)
{
+ // TODO-CQ: If the destination is an aggregate we can still use liveness
+ // information for the remainder to DCE this.
return;
}
DecompositionStatementList result;
EliminateCommasInBlockOp(asg, &result);
+ DecompositionPlan plan(m_compiler, m_aggregates, m_liveness, dst, src, dstInvolvesReplacements,
+ srcInvolvesReplacements);
+
if (dstInvolvesReplacements)
{
unsigned dstLclOffs = dstLcl->GetLclOffs();
dstFirstRep->NeedsWriteBack = false;
}
+ plan.MarkNonRemainderUseOfStructLocal();
dstFirstRep->NeedsReadBack = true;
dstFirstRep++;
}
dstLastRep->NeedsWriteBack = false;
}
+ plan.MarkNonRemainderUseOfStructLocal();
dstLastRep->NeedsReadBack = true;
dstEndRep--;
}
}
}
- DecompositionPlan plan(m_compiler, dst, src, srcInvolvesReplacements);
-
if (src->IsConstInitVal())
{
InitFields(dst->AsLclVarCommon(), dstFirstRep, dstEndRep, &plan);
Replacement** firstReplacement,
Replacement** endReplacement)
{
- if (m_replacements[lcl->GetLclNum()] == nullptr)
+ AggregateInfo* agg = m_aggregates[lcl->GetLclNum()];
+ if (agg == nullptr)
{
return false;
}
- jitstd::vector<Replacement>& replacements = *m_replacements[lcl->GetLclNum()];
-
- unsigned offs = lcl->GetLclOffs();
- unsigned size = lcl->GetLayout(m_compiler)->GetSize();
- size_t firstIndex = Promotion::BinarySearch<Replacement, &Replacement::Offset>(replacements, offs);
- if ((ssize_t)firstIndex < 0)
- {
- firstIndex = ~firstIndex;
- if (firstIndex > 0)
- {
- Replacement& lastRepBefore = replacements[firstIndex - 1];
- if ((lastRepBefore.Offset + genTypeSize(lastRepBefore.AccessType)) > offs)
- {
- // Overlap with last entry starting before offs.
- firstIndex--;
- }
- else if (firstIndex >= replacements.size())
- {
- // Starts after last replacement ends.
- return false;
- }
- }
-
- const Replacement& first = replacements[firstIndex];
- if (first.Offset >= (offs + size))
- {
- // First candidate starts after this ends.
- return false;
- }
- }
-
- assert((firstIndex < replacements.size()) && replacements[firstIndex].Overlaps(offs, size));
- *firstReplacement = &replacements[firstIndex];
-
- if (endReplacement != nullptr)
- {
- size_t lastIndex = Promotion::BinarySearch<Replacement, &Replacement::Offset>(replacements, offs + size);
- if ((ssize_t)lastIndex < 0)
- {
- lastIndex = ~lastIndex;
- }
-
- // Since we verified above that there is an overlapping replacement
- // we know that lastIndex exists and is the next one that does not
- // overlap.
- assert(lastIndex > 0);
- *endReplacement = replacements.data() + lastIndex;
- }
-
- return true;
+ unsigned offs = lcl->GetLclOffs();
+ unsigned size = lcl->GetLayout(m_compiler)->GetSize();
+ return agg->OverlappingReplacements(offs, size, firstReplacement, endReplacement);
}
//------------------------------------------------------------------------
// We will need to read this one back after initing the struct.
rep->NeedsWriteBack = false;
rep->NeedsReadBack = true;
+ plan->MarkNonRemainderUseOfStructLocal();
continue;
}
}
}
+#ifdef DEBUG
+//------------------------------------------------------------------------
+// LastUseString:
+// Return a string indicating whether a replacement is a last use, for
+// JITDUMP purposes.
+//
+// Parameters:
+// lcl - A struct local
+// rep - A replacement of that struct local
+//
+// Returns:
+// " (last use)" if it is, and otherwise "".
+//
+const char* ReplaceVisitor::LastUseString(GenTreeLclVarCommon* lcl, Replacement* rep)
+{
+ StructDeaths deaths = m_liveness->GetDeathsForStructLocal(lcl);
+ AggregateInfo* agg = m_aggregates[lcl->GetLclNum()];
+ assert(agg != nullptr);
+ Replacement* firstRep = agg->Replacements.data();
+ assert((rep >= firstRep) && (rep < firstRep + agg->Replacements.size()));
+
+ size_t replacementIndex = rep - firstRep;
+ if (deaths.IsReplacementDying((unsigned)replacementIndex))
+ {
+ return " (last use)";
+ }
+
+ return "";
+}
+#endif
+
//------------------------------------------------------------------------
// CopyBetweenFields:
// Copy between two struct locals that may involve replacements.
// Write it directly to the destination struct local.
unsigned offs = srcRep->Offset - srcBaseOffs;
plan->CopyFromReplacement(srcRep, offs);
- JITDUMP(" dst+%03u <- V%02u (%s)\n", offs, srcRep->LclNum, srcRep->Description);
+ JITDUMP(" dst+%03u <- V%02u (%s)%s\n", offs, srcRep->LclNum, srcRep->Description,
+ LastUseString(srcLcl, srcRep));
srcRep++;
continue;
}
// Read it directly from the source struct local.
unsigned offs = dstRep->Offset - dstBaseOffs;
plan->CopyToReplacement(dstRep, offs);
- JITDUMP(" V%02u (%s) <- src+%03u\n", dstRep->LclNum, dstRep->Description, offs);
+ JITDUMP(" V%02u (%s)%s <- src+%03u\n", dstRep->LclNum, dstRep->Description,
+ LastUseString(dstLcl, dstRep), offs);
dstRep++;
continue;
}
(dstRep->AccessType == srcRep->AccessType))
{
plan->CopyBetweenReplacements(dstRep, srcRep, dstRep->Offset - dstBaseOffs);
- JITDUMP(" V%02u (%s) <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description, srcRep->LclNum,
- srcRep->Description);
+ JITDUMP(" V%02u (%s)%s <- V%02u (%s)%s\n", dstRep->LclNum, dstRep->Description,
+ LastUseString(dstLcl, dstRep), srcRep->LclNum, srcRep->Description,
+ LastUseString(srcLcl, srcRep));
dstRep++;
srcRep++;
// will handle the destination replacement in a future
// iteration of the loop.
statements->AddStatement(Promotion::CreateWriteBack(m_compiler, srcLcl->GetLclNum(), *srcRep));
- JITDUMP(" Partial overlap of V%02u (%s) <- V%02u (%s). Will read source back before copy\n",
- dstRep->LclNum, dstRep->Description, srcRep->LclNum, srcRep->Description);
+ JITDUMP(" Partial overlap of V%02u (%s)%s <- V%02u (%s)%s. Will read source back before copy\n",
+ dstRep->LclNum, dstRep->Description, LastUseString(dstLcl, dstRep), srcRep->LclNum,
+ srcRep->Description, LastUseString(srcLcl, srcRep));
srcRep++;
continue;
}
if (dsc->lvType == dstRep->AccessType)
{
plan->CopyBetweenReplacements(dstRep, fieldLcl, offs);
- JITDUMP(" V%02u (%s) <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description, dsc->lvReason);
+ JITDUMP(" V%02u (%s)%s <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description,
+ LastUseString(dstLcl, dstRep), dsc->lvReason);
dstRep++;
continue;
}
// directly to the destination's struct local and mark the
// overlapping fields as needing read back to avoid this DNER.
plan->CopyToReplacement(dstRep, offs);
- JITDUMP(" V%02u (%s) <- src+%03u\n", dstRep->LclNum, dstRep->Description, offs);
+ JITDUMP(" V%02u (%s)%s <- src+%03u\n", dstRep->LclNum, dstRep->Description, LastUseString(dstLcl, dstRep),
+ offs);
dstRep++;
}
else
if (dsc->lvType == srcRep->AccessType)
{
plan->CopyBetweenReplacements(fieldLcl, srcRep, offs);
- JITDUMP(" V%02u (%s) <- V%02u (%s)\n", fieldLcl, dsc->lvReason, srcRep->LclNum,
- srcRep->Description);
+ JITDUMP(" V%02u (%s) <- V%02u (%s)%s\n", fieldLcl, dsc->lvReason, srcRep->LclNum,
+ srcRep->Description, LastUseString(srcLcl, srcRep));
srcRep++;
continue;
}
}
plan->CopyFromReplacement(srcRep, offs);
- JITDUMP(" dst+%03u <- V%02u (%s)\n", offs, srcRep->LclNum, srcRep->Description);
+ JITDUMP(" dst+%03u <- V%02u (%s)%s\n", offs, srcRep->LclNum, srcRep->Description,
+ LastUseString(srcLcl, srcRep));
srcRep++;
}
}
--- /dev/null
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "jitpch.h"
+#include "promotion.h"
+
+struct BasicBlockLiveness
+{
+ // Variables used before a full definition.
+ BitVec VarUse;
+ // Variables fully defined before a use.
+ // Note that this differs from our normal liveness: partial definitions are
+ // NOT marked but they are also not considered uses.
+ BitVec VarDef;
+ // Variables live-in to this basic block.
+ BitVec LiveIn;
+ // Variables live-out of this basic block.
+ BitVec LiveOut;
+};
+
+//------------------------------------------------------------------------
+// Run:
+// Compute liveness information pertaining the promoted structs.
+//
+// Remarks:
+// For each promoted aggregate we compute the liveness for its remainder and
+// all of its fields. Unlike regular liveness we currently do not do any DCE
+// here and so only do the dataflow computation once.
+//
+// The liveness information is written into the IR using the normal
+// GTF_VAR_DEATH flag. Note that the semantics of GTF_VAR_DEATH differs from
+// the rest of the JIT for a short while between the liveness is computed and
+// the replacement phase has run: in particular, after this liveness pass you
+// may see a node like:
+//
+// LCL_FLD int V16 tmp9 [+8] (last use)
+//
+// that indicates that this particular field (or the remainder if it wasn't
+// promoted) is dying, not that V16 itself is dying. After replacement has
+// run the semantics align with the rest of the JIT: in the promoted case V16
+// [+8] would be replaced by its promoted field local, and in the remainder
+// case all non-remainder uses of V16 would also be.
+//
+// There is one catch which is struct uses of the local. These can indicate
+// deaths of multiple fields and also the remainder, so this information is
+// stored on the side. PromotionLiveness::GetDeathsForStructLocal is used to
+// query this information.
+//
+// The liveness information is used by decomposition to avoid creating dead
+// stores, and also to mark the replacement field uses/defs with proper
+// up-to-date liveness information to be used by future phases (forward sub
+// and morph, as of writing this). It is also used to avoid creating
+// unnecessary read-backs; this is mostly just a TP optimization as future
+// liveness passes would be expected to DCE these anyway.
+//
+// Avoiding the creation of dead stores to the remainder is especially
+// important as these otherwise would often end up looking like partial
+// definitions, and the other liveness passes handle partial definitions very
+// conservatively and are not able to DCE them.
+//
+// Unlike the other liveness passes we keep the per-block liveness
+// information on the side and we do not update BasicBlock::bbLiveIn et al.
+// This relies on downstream phases not requiring/wanting to use per-basic
+// block live-in/live-out/var-use/var-def sets. To be able to update these we
+// would need to give the new locals "regular" tracked indices (i.e. allocate
+// a lvVarIndex).
+//
+// The indices allocated and used internally within the liveness computation
+// are "dense" in the sense that the bit vectors only have indices for
+// remainders and the replacement fields introduced by this pass. In other
+// words, we allocate 1 + num_fields indices for each promoted struct local).
+//
+void PromotionLiveness::Run()
+{
+ m_structLclToTrackedIndex = new (m_compiler, CMK_Promotion) unsigned[m_aggregates.size()]{};
+ unsigned trackedIndex = 0;
+ for (size_t lclNum = 0; lclNum < m_aggregates.size(); lclNum++)
+ {
+ AggregateInfo* agg = m_aggregates[lclNum];
+ if (agg == nullptr)
+ {
+ continue;
+ }
+
+ m_structLclToTrackedIndex[lclNum] = trackedIndex;
+ // TODO: We need a scalability limit on these, we cannot always track
+ // the remainder and all fields.
+ // Remainder.
+ trackedIndex++;
+ // Fields.
+ trackedIndex += (unsigned)agg->Replacements.size();
+
+#ifdef DEBUG
+ // Mark the struct local (remainder) and fields as tracked for DISPTREE to properly
+ // show last use information.
+ m_compiler->lvaGetDesc((unsigned)lclNum)->lvTrackedWithoutIndex = true;
+ for (size_t i = 0; i < agg->Replacements.size(); i++)
+ {
+ m_compiler->lvaGetDesc(agg->Replacements[i].LclNum)->lvTrackedWithoutIndex = true;
+ }
+#endif
+ }
+
+ m_numVars = trackedIndex;
+
+ m_bvTraits = new (m_compiler, CMK_Promotion) BitVecTraits(m_numVars, m_compiler);
+ m_bbInfo = m_compiler->fgAllocateTypeForEachBlk<BasicBlockLiveness>(CMK_Promotion);
+ BitVecOps::AssignNoCopy(m_bvTraits, m_liveIn, BitVecOps::MakeEmpty(m_bvTraits));
+ BitVecOps::AssignNoCopy(m_bvTraits, m_ehLiveVars, BitVecOps::MakeEmpty(m_bvTraits));
+
+ JITDUMP("Computing liveness for %u remainders/fields\n\n", m_numVars);
+
+ ComputeUseDefSets();
+
+ InterBlockLiveness();
+
+ FillInLiveness();
+}
+
+//------------------------------------------------------------------------
+// ComputeUseDefSets:
+// Compute the use and def sets for all blocks.
+//
+void PromotionLiveness::ComputeUseDefSets()
+{
+ for (BasicBlock* block : m_compiler->Blocks())
+ {
+ BasicBlockLiveness& bb = m_bbInfo[block->bbNum];
+ BitVecOps::AssignNoCopy(m_bvTraits, bb.VarUse, BitVecOps::MakeEmpty(m_bvTraits));
+ BitVecOps::AssignNoCopy(m_bvTraits, bb.VarDef, BitVecOps::MakeEmpty(m_bvTraits));
+ BitVecOps::AssignNoCopy(m_bvTraits, bb.LiveIn, BitVecOps::MakeEmpty(m_bvTraits));
+ BitVecOps::AssignNoCopy(m_bvTraits, bb.LiveOut, BitVecOps::MakeEmpty(m_bvTraits));
+
+ if (m_compiler->compQmarkUsed)
+ {
+ for (Statement* stmt : block->Statements())
+ {
+ GenTree* dst;
+ GenTree* qmark = m_compiler->fgGetTopLevelQmark(stmt->GetRootNode(), &dst);
+ if (qmark == nullptr)
+ {
+ for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList())
+ {
+ MarkUseDef(lcl, bb.VarUse, bb.VarDef);
+ }
+ }
+ else
+ {
+ for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList())
+ {
+ // Skip liveness updates/marking for defs; they may be conditionally executed.
+ if ((lcl->gtFlags & GTF_VAR_DEF) == 0)
+ {
+ MarkUseDef(lcl, bb.VarUse, bb.VarDef);
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ for (Statement* stmt : block->Statements())
+ {
+ for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList())
+ {
+ MarkUseDef(lcl, bb.VarUse, bb.VarDef);
+ }
+ }
+ }
+
+#ifdef DEBUG
+ if (m_compiler->verbose)
+ {
+ BitVec allVars(BitVecOps::Union(m_bvTraits, bb.VarUse, bb.VarDef));
+ printf(FMT_BB " USE(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bb.VarUse));
+ DumpVarSet(bb.VarUse, allVars);
+ printf("\n" FMT_BB " DEF(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bb.VarDef));
+ DumpVarSet(bb.VarDef, allVars);
+ printf("\n\n");
+ }
+#endif
+ }
+}
+
+//------------------------------------------------------------------------
+// MarkUseDef:
+// Mark use/def information for a single appearence of a local.
+//
+// Parameters:
+// lcl - The local node
+// useSet - The use set to mark in.
+// defSet - The def set to mark in.
+//
+void PromotionLiveness::MarkUseDef(GenTreeLclVarCommon* lcl, BitSetShortLongRep& useSet, BitSetShortLongRep& defSet)
+{
+ AggregateInfo* agg = m_aggregates[lcl->GetLclNum()];
+ if (agg == nullptr)
+ {
+ return;
+ }
+
+ jitstd::vector<Replacement>& reps = agg->Replacements;
+ bool isDef = (lcl->gtFlags & GTF_VAR_DEF) != 0;
+ bool isUse = !isDef;
+
+ unsigned baseIndex = m_structLclToTrackedIndex[lcl->GetLclNum()];
+ var_types accessType = lcl->TypeGet();
+
+ if (accessType == TYP_STRUCT)
+ {
+ if (lcl->OperIs(GT_LCL_ADDR))
+ {
+ // For LCL_ADDR this is a retbuf and we expect it to be a def. We
+ // don't know the exact size here so we cannot mark anything as
+ // being fully defined, thus we can just return.
+ assert(isDef);
+ return;
+ }
+
+ if (lcl->OperIsScalarLocal())
+ {
+ // Mark remainder and all fields.
+ for (size_t i = 0; i <= reps.size(); i++)
+ {
+ MarkIndex(baseIndex + (unsigned)i, isUse, isDef, useSet, defSet);
+ }
+ }
+ else
+ {
+ unsigned offs = lcl->GetLclOffs();
+ unsigned size = lcl->GetLayout(m_compiler)->GetSize();
+ size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(reps, offs);
+
+ if ((ssize_t)index < 0)
+ {
+ index = ~index;
+ if ((index > 0) && reps[index - 1].Overlaps(offs, size))
+ {
+ index--;
+ }
+ }
+
+ while ((index < reps.size()) && (reps[index].Offset < offs + size))
+ {
+ Replacement& rep = reps[index];
+ bool isFullFieldDef =
+ isDef && (offs <= rep.Offset) && (offs + size >= rep.Offset + genTypeSize(rep.AccessType));
+ MarkIndex(baseIndex + 1 + (unsigned)index, isUse, isFullFieldDef, useSet, defSet);
+ index++;
+ }
+
+ bool isFullDefOfRemainder = isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size));
+ // TODO-CQ: We could also try to figure out if a use actually touches the remainder, e.g. in some cases
+ // a struct use may consist only of promoted fields and does not actually use the remainder.
+ MarkIndex(baseIndex, isUse, isFullDefOfRemainder, useSet, defSet);
+ }
+ }
+ else
+ {
+ unsigned offs = lcl->GetLclOffs();
+ size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(reps, offs);
+ if ((ssize_t)index < 0)
+ {
+ unsigned size = genTypeSize(accessType);
+ bool isFullDefOfRemainder = isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size));
+ MarkIndex(baseIndex, isUse, isFullDefOfRemainder, useSet, defSet);
+ }
+ else
+ {
+ // Accessing element.
+ MarkIndex(baseIndex + 1 + (unsigned)index, isUse, isDef, useSet, defSet);
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// MarkIndex:
+// Mark specific bits in use/def bit vectors depending on whether this is a use def.
+//
+// Parameters:
+// index - The index of the bit to set.
+// isUse - Whether this is a use
+// isDef - Whether this is a def
+// useSet - The set of uses
+// defSet - The set of defs
+//
+void PromotionLiveness::MarkIndex(unsigned index, bool isUse, bool isDef, BitVec& useSet, BitVec& defSet)
+{
+ if (isUse && !BitVecOps::IsMember(m_bvTraits, defSet, index))
+ {
+ BitVecOps::AddElemD(m_bvTraits, useSet, index);
+ }
+
+ if (isDef)
+ {
+ BitVecOps::AddElemD(m_bvTraits, defSet, index);
+ }
+}
+
+//------------------------------------------------------------------------
+// InterBlockLiveness:
+// Compute the fixpoint.
+//
+void PromotionLiveness::InterBlockLiveness()
+{
+ bool changed;
+ do
+ {
+ changed = false;
+
+ for (BasicBlock* block = m_compiler->fgLastBB; block != nullptr; block = block->bbPrev)
+ {
+ m_hasPossibleBackEdge |= block->bbNext && (block->bbNext->bbNum <= block->bbNum);
+ changed |= PerBlockLiveness(block);
+ }
+
+ if (!m_hasPossibleBackEdge)
+ {
+ break;
+ }
+ } while (changed);
+
+#ifdef DEBUG
+ if (m_compiler->verbose)
+ {
+ for (BasicBlock* block : m_compiler->Blocks())
+ {
+ BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum];
+ BitVec allVars(BitVecOps::Union(m_bvTraits, bbInfo.LiveIn, bbInfo.LiveOut));
+ printf(FMT_BB " IN (%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bbInfo.LiveIn));
+ DumpVarSet(bbInfo.LiveIn, allVars);
+ printf("\n" FMT_BB " OUT(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bbInfo.LiveOut));
+ DumpVarSet(bbInfo.LiveOut, allVars);
+ printf("\n\n");
+ }
+ }
+#endif
+}
+
+//------------------------------------------------------------------------
+// PerBlockLiveness:
+// Compute liveness for a single block during a single iteration of the
+// fixpoint computation.
+//
+// Parameters:
+// block - The block
+//
+bool PromotionLiveness::PerBlockLiveness(BasicBlock* block)
+{
+ // We disable promotion for GT_JMP methods.
+ assert(!block->endsWithJmpMethod(m_compiler));
+
+ BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum];
+ BitVecOps::ClearD(m_bvTraits, bbInfo.LiveOut);
+ for (BasicBlock* succ : block->GetAllSuccs(m_compiler))
+ {
+ BitVecOps::UnionD(m_bvTraits, bbInfo.LiveOut, m_bbInfo[succ->bbNum].LiveIn);
+ m_hasPossibleBackEdge |= succ->bbNum <= block->bbNum;
+ }
+
+ BitVecOps::LivenessD(m_bvTraits, m_liveIn, bbInfo.VarDef, bbInfo.VarUse, bbInfo.LiveOut);
+
+ if (m_compiler->ehBlockHasExnFlowDsc(block))
+ {
+ BitVecOps::ClearD(m_bvTraits, m_ehLiveVars);
+ AddHandlerLiveVars(block, m_ehLiveVars);
+ BitVecOps::UnionD(m_bvTraits, m_liveIn, m_ehLiveVars);
+ BitVecOps::UnionD(m_bvTraits, bbInfo.LiveOut, m_ehLiveVars);
+ m_hasPossibleBackEdge = true;
+ }
+
+ bool liveInChanged = !BitVecOps::Equal(m_bvTraits, bbInfo.LiveIn, m_liveIn);
+
+ if (liveInChanged)
+ {
+ BitVecOps::Assign(m_bvTraits, bbInfo.LiveIn, m_liveIn);
+ }
+
+ return liveInChanged;
+}
+
+//------------------------------------------------------------------------
+// AddHandlerLiveVars:
+// Find variables that are live-in to handlers reachable by implicit control
+// flow and add them to a specified bit vector.
+//
+// Parameters:
+// block - The block
+// ehLiveVars - The bit vector to mark in
+//
+// Remarks:
+// Similar to Compiler::fgGetHandlerLiveVars used by regular liveness.
+//
+void PromotionLiveness::AddHandlerLiveVars(BasicBlock* block, BitVec& ehLiveVars)
+{
+ assert(m_compiler->ehBlockHasExnFlowDsc(block));
+ EHblkDsc* HBtab = m_compiler->ehGetBlockExnFlowDsc(block);
+
+ do
+ {
+ // Either we enter the filter first or the catch/finally
+ if (HBtab->HasFilter())
+ {
+ BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdFilter->bbNum].LiveIn);
+#if defined(FEATURE_EH_FUNCLETS)
+ // The EH subsystem can trigger a stack walk after the filter
+ // has returned, but before invoking the handler, and the only
+ // IP address reported from this method will be the original
+ // faulting instruction, thus everything in the try body
+ // must report as live any variables live-out of the filter
+ // (which is the same as those live-in to the handler)
+ BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdHndBeg->bbNum].LiveIn);
+#endif // FEATURE_EH_FUNCLETS
+ }
+ else
+ {
+ BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdHndBeg->bbNum].LiveIn);
+ }
+
+ // If we have nested try's edbEnclosing will provide them
+ assert((HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ||
+ (HBtab->ebdEnclosingTryIndex > m_compiler->ehGetIndex(HBtab)));
+
+ unsigned outerIndex = HBtab->ebdEnclosingTryIndex;
+ if (outerIndex == EHblkDsc::NO_ENCLOSING_INDEX)
+ {
+ break;
+ }
+ HBtab = m_compiler->ehGetDsc(outerIndex);
+
+ } while (true);
+
+ // If this block is within a filter, we also need to report as live
+ // any vars live into enclosed finally or fault handlers, since the
+ // filter will run during the first EH pass, and enclosed or enclosing
+ // handlers will run during the second EH pass. So all these handlers
+ // are "exception flow" successors of the filter.
+ //
+ // Note we are relying on ehBlockHasExnFlowDsc to return true
+ // for any filter block that we should examine here.
+ if (block->hasHndIndex())
+ {
+ const unsigned thisHndIndex = block->getHndIndex();
+ EHblkDsc* enclosingHBtab = m_compiler->ehGetDsc(thisHndIndex);
+
+ if (enclosingHBtab->InFilterRegionBBRange(block))
+ {
+ assert(enclosingHBtab->HasFilter());
+
+ // Search the EH table for enclosed regions.
+ //
+ // All the enclosed regions will be lower numbered and
+ // immediately prior to and contiguous with the enclosing
+ // region in the EH tab.
+ unsigned index = thisHndIndex;
+
+ while (index > 0)
+ {
+ index--;
+ unsigned enclosingIndex = m_compiler->ehGetEnclosingTryIndex(index);
+ bool isEnclosed = false;
+
+ // To verify this is an enclosed region, search up
+ // through the enclosing regions until we find the
+ // region associated with the filter.
+ while (enclosingIndex != EHblkDsc::NO_ENCLOSING_INDEX)
+ {
+ if (enclosingIndex == thisHndIndex)
+ {
+ isEnclosed = true;
+ break;
+ }
+
+ enclosingIndex = m_compiler->ehGetEnclosingTryIndex(enclosingIndex);
+ }
+
+ // If we found an enclosed region, check if the region
+ // is a try fault or try finally, and if so, add any
+ // locals live into the enclosed region's handler into this
+ // block's live-in set.
+ if (isEnclosed)
+ {
+ EHblkDsc* enclosedHBtab = m_compiler->ehGetDsc(index);
+
+ if (enclosedHBtab->HasFinallyOrFaultHandler())
+ {
+ BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[enclosedHBtab->ebdHndBeg->bbNum].LiveIn);
+ }
+ }
+ // Once we run across a non-enclosed region, we can stop searching.
+ else
+ {
+ break;
+ }
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// FillInLiveness:
+// Starting with the live-out set for each basic block do a backwards traversal
+// marking liveness into the IR.
+//
+void PromotionLiveness::FillInLiveness()
+{
+ BitVec life(BitVecOps::MakeEmpty(m_bvTraits));
+ BitVec volatileVars(BitVecOps::MakeEmpty(m_bvTraits));
+
+ for (BasicBlock* block : m_compiler->Blocks())
+ {
+ if (block->firstStmt() == nullptr)
+ {
+ continue;
+ }
+
+ BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum];
+
+ BitVecOps::ClearD(m_bvTraits, volatileVars);
+
+ if (m_compiler->ehBlockHasExnFlowDsc(block))
+ {
+ AddHandlerLiveVars(block, volatileVars);
+ }
+
+ BitVecOps::Assign(m_bvTraits, life, bbInfo.LiveOut);
+
+ Statement* stmt = block->lastStmt();
+
+ while (true)
+ {
+ GenTree* qmark = nullptr;
+ if (m_compiler->compQmarkUsed)
+ {
+ GenTree* dst;
+ qmark = m_compiler->fgGetTopLevelQmark(stmt->GetRootNode(), &dst);
+ }
+
+ if (qmark == nullptr)
+ {
+ for (GenTree* cur = stmt->GetTreeListEnd(); cur != nullptr; cur = cur->gtPrev)
+ {
+ FillInLiveness(life, volatileVars, cur->AsLclVarCommon());
+ }
+ }
+ else
+ {
+ for (GenTree* cur = stmt->GetTreeListEnd(); cur != nullptr; cur = cur->gtPrev)
+ {
+ // Skip liveness updates/marking for defs; they may be conditionally executed.
+ if ((cur->gtFlags & GTF_VAR_DEF) == 0)
+ {
+ FillInLiveness(life, volatileVars, cur->AsLclVarCommon());
+ }
+ }
+ }
+
+ if (stmt == block->firstStmt())
+ {
+ break;
+ }
+
+ stmt = stmt->GetPrevStmt();
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// FillInLiveness:
+// Fill liveness information into the specified IR node.
+//
+// Parameters:
+// life - The current life set. Will be read and updated depending on 'lcl'.
+// volatileVars - Bit vector of variables that are live always.
+// lcl - The IR node to process liveness for and to mark with liveness information.
+//
+void PromotionLiveness::FillInLiveness(BitVec& life, BitVec volatileVars, GenTreeLclVarCommon* lcl)
+{
+ AggregateInfo* agg = m_aggregates[lcl->GetLclNum()];
+ if (agg == nullptr)
+ {
+ return;
+ }
+
+ bool isDef = (lcl->gtFlags & GTF_VAR_DEF) != 0;
+ bool isUse = !isDef;
+
+ unsigned baseIndex = m_structLclToTrackedIndex[lcl->GetLclNum()];
+ var_types accessType = lcl->TypeGet();
+
+ if (accessType == TYP_STRUCT)
+ {
+ // We need an external bit set to represent dying fields/remainder on a struct use.
+ BitVecTraits aggTraits(1 + (unsigned)agg->Replacements.size(), m_compiler);
+ BitVec aggDeaths(BitVecOps::MakeEmpty(&aggTraits));
+ if (lcl->OperIsScalarLocal())
+ {
+ // Handle remainder and all fields.
+ for (size_t i = 0; i <= agg->Replacements.size(); i++)
+ {
+ unsigned varIndex = baseIndex + (unsigned)i;
+ if (BitVecOps::IsMember(m_bvTraits, life, varIndex))
+ {
+ if (isDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex))
+ {
+ BitVecOps::RemoveElemD(m_bvTraits, life, varIndex);
+ }
+ }
+ else
+ {
+ BitVecOps::AddElemD(&aggTraits, aggDeaths, (unsigned)i);
+
+ if (isUse)
+ {
+ BitVecOps::AddElemD(m_bvTraits, life, varIndex);
+ }
+ }
+ }
+ }
+ else
+ {
+ unsigned offs = lcl->GetLclOffs();
+ unsigned size = lcl->GetLayout(m_compiler)->GetSize();
+ size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(agg->Replacements, offs);
+
+ if ((ssize_t)index < 0)
+ {
+ index = ~index;
+ if ((index > 0) && agg->Replacements[index - 1].Overlaps(offs, size))
+ {
+ index--;
+ }
+ }
+
+ // Handle fields.
+ while ((index < agg->Replacements.size()) && (agg->Replacements[index].Offset < offs + size))
+ {
+ unsigned varIndex = baseIndex + 1 + (unsigned)index;
+ Replacement& rep = agg->Replacements[index];
+ if (BitVecOps::IsMember(m_bvTraits, life, varIndex))
+ {
+ bool isFullFieldDef =
+ isDef && (offs <= rep.Offset) && (offs + size >= rep.Offset + genTypeSize(rep.AccessType));
+ if (isFullFieldDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex))
+ {
+ BitVecOps::RemoveElemD(m_bvTraits, life, varIndex);
+ }
+ }
+ else
+ {
+ BitVecOps::AddElemD(&aggTraits, aggDeaths, 1 + (unsigned)index);
+
+ if (isUse)
+ {
+ BitVecOps::AddElemD(m_bvTraits, life, varIndex);
+ }
+ }
+
+ index++;
+ }
+
+ // Handle remainder.
+ if (BitVecOps::IsMember(m_bvTraits, life, baseIndex))
+ {
+ bool isFullDefOfRemainder =
+ isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size));
+ if (isFullDefOfRemainder && !BitVecOps::IsMember(m_bvTraits, volatileVars, baseIndex))
+ {
+ BitVecOps::RemoveElemD(m_bvTraits, life, baseIndex);
+ }
+ }
+ else
+ {
+ // TODO-CQ: We could also try to figure out if a use actually touches the remainder, e.g. in some cases
+ // a struct use may consist only of promoted fields and does not actually use the remainder.
+ BitVecOps::AddElemD(&aggTraits, aggDeaths, 0);
+
+ if (isUse)
+ {
+ BitVecOps::AddElemD(m_bvTraits, life, baseIndex);
+ }
+ }
+ }
+
+ m_aggDeaths.Set(lcl, aggDeaths);
+ }
+ else
+ {
+ if (lcl->OperIs(GT_LCL_ADDR))
+ {
+ // Retbuf -- these are definitions but we do not know of how much.
+ // We never mark them as dead and we never treat them as killing anything.
+ assert(isDef);
+ return;
+ }
+
+ unsigned offs = lcl->GetLclOffs();
+ size_t index = Promotion::BinarySearch<Replacement, &Replacement::Offset>(agg->Replacements, offs);
+ if ((ssize_t)index < 0)
+ {
+ // No replacement found, this is a use of the remainder.
+ unsigned size = genTypeSize(accessType);
+ if (BitVecOps::IsMember(m_bvTraits, life, baseIndex))
+ {
+ lcl->gtFlags &= ~GTF_VAR_DEATH;
+
+ bool isFullDefOfRemainder =
+ isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size));
+ if (isFullDefOfRemainder && !BitVecOps::IsMember(m_bvTraits, volatileVars, baseIndex))
+ {
+ BitVecOps::RemoveElemD(m_bvTraits, life, baseIndex);
+ }
+ }
+ else
+ {
+ lcl->gtFlags |= GTF_VAR_DEATH;
+
+ if (isUse)
+ {
+ BitVecOps::AddElemD(m_bvTraits, life, baseIndex);
+ }
+ }
+ }
+ else
+ {
+ // Use of a field.
+ unsigned varIndex = baseIndex + 1 + (unsigned)index;
+
+ if (BitVecOps::IsMember(m_bvTraits, life, varIndex))
+ {
+ lcl->gtFlags &= ~GTF_VAR_DEATH;
+
+ if (isDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex))
+ {
+ BitVecOps::RemoveElemD(m_bvTraits, life, varIndex);
+ }
+ }
+ else
+ {
+ lcl->gtFlags |= GTF_VAR_DEATH;
+
+ if (isUse)
+ {
+ BitVecOps::AddElemD(m_bvTraits, life, varIndex);
+ }
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// IsReplacementLiveOut:
+// Check if a replacement field is live at the end of a basic block.
+//
+// Parameters:
+// structLcl - The struct (base) local
+// replacementIndex - Index of the replacement
+//
+// Returns:
+// True if the field is in the live-out set.
+//
+bool PromotionLiveness::IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacementIndex)
+{
+ BitVec liveOut = m_bbInfo[bb->bbNum].LiveOut;
+ unsigned baseIndex = m_structLclToTrackedIndex[structLcl];
+ return BitVecOps::IsMember(m_bvTraits, liveOut, baseIndex + 1 + replacementIndex);
+}
+
+//------------------------------------------------------------------------
+// GetDeathsForStructLocal:
+// Get a data structure that can be used to query liveness information
+// for a specified local node at its position.
+//
+// Parameters:
+// lcl - The node
+//
+// Returns:
+// Liveness information.
+//
+StructDeaths PromotionLiveness::GetDeathsForStructLocal(GenTreeLclVarCommon* lcl)
+{
+ assert(lcl->OperIsLocal() && lcl->TypeIs(TYP_STRUCT) && (m_aggregates[lcl->GetLclNum()] != nullptr));
+ BitVec aggDeaths;
+ bool found = m_aggDeaths.Lookup(lcl, &aggDeaths);
+ assert(found);
+
+ unsigned lclNum = lcl->GetLclNum();
+ AggregateInfo* aggInfo = m_aggregates[lclNum];
+ return StructDeaths(aggDeaths, (unsigned)aggInfo->Replacements.size());
+}
+
+//------------------------------------------------------------------------
+// IsRemainderDying:
+// Check if the remainder is dying.
+//
+// Returns:
+// True if so.
+//
+bool StructDeaths::IsRemainderDying() const
+{
+ BitVecTraits traits(1 + m_numFields, nullptr);
+ return BitVecOps::IsMember(&traits, m_deaths, 0);
+}
+
+//------------------------------------------------------------------------
+// IsReplacementDying:
+// Check if a specific replacement is dying.
+//
+// Returns:
+// True if so.
+//
+bool StructDeaths::IsReplacementDying(unsigned index) const
+{
+ BitVecTraits traits(1 + m_numFields, nullptr);
+ return BitVecOps::IsMember(&traits, m_deaths, 1 + index);
+}
+
+#ifdef DEBUG
+//------------------------------------------------------------------------
+// DumpVarSet:
+// Dump a var set to jitstdout.
+//
+// Parameters:
+// set - The set to dump
+// allVars - Set of all variables to print whitespace for if not in 'set'.
+// Used for alignment.
+//
+void PromotionLiveness::DumpVarSet(BitVec set, BitVec allVars)
+{
+ printf("{");
+
+ const char* sep = "";
+ for (size_t i = 0; i < m_aggregates.size(); i++)
+ {
+ AggregateInfo* agg = m_aggregates[i];
+ if (agg == nullptr)
+ {
+ continue;
+ }
+
+ for (size_t j = 0; j <= agg->Replacements.size(); j++)
+ {
+ unsigned index = (unsigned)(m_structLclToTrackedIndex[i] + j);
+
+ if (BitVecOps::IsMember(m_bvTraits, set, index))
+ {
+ if (j == 0)
+ {
+ printf("%sV%02u(remainder)", sep, (unsigned)i);
+ }
+ else
+ {
+ const Replacement& rep = agg->Replacements[j - 1];
+ printf("%sV%02u.[%03u..%03u)", sep, (unsigned)i, rep.Offset,
+ rep.Offset + genTypeSize(rep.AccessType));
+ }
+ sep = " ";
+ }
+ else if (BitVecOps::IsMember(m_bvTraits, allVars, index))
+ {
+ printf("%s ", sep);
+ sep = " ";
+ }
+ }
+ }
+
+ printf("}");
+}
+#endif
projects / platform / upstream / dotnet / runtime.git / commitdiff