From: Jakob Botsch Nielsen Date: Fri, 19 May 2023 20:07:48 +0000 (+0200) Subject: JIT: Add a pass of liveness for the new locals inside physical promotion (#86043) X-Git-Tag: accepted/tizen/unified/riscv/20231226.055536~2090 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=e62cb64335549d48439585d436a4903ab06136a3;p=platform%2Fupstream%2Fdotnet%2Fruntime.git JIT: Add a pass of liveness for the new locals inside physical promotion (#86043) Used when decomposing stores to skip the fields that are dying and the remainder if it is dying. Also allows forward sub to kick in for the inserted replacement locals. --- diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index d1af5c5..df18f4c 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -160,6 +160,7 @@ set( JIT_SOURCES phase.cpp promotion.cpp promotiondecomposition.cpp + promotionliveness.cpp rangecheck.cpp rationalize.cpp redundantbranchopts.cpp diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index a342604..fc02278 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -513,6 +513,10 @@ public: { return lvTracked && lvType != TYP_STRUCT; } +#ifdef DEBUG + unsigned char lvTrackedWithoutIndex : 1; // Tracked but has no lvVarIndex (i.e. only valid GTF_VAR_DEATH flags, used + // by physical promotion) +#endif unsigned char lvPinned : 1; // is this a pinned variable? unsigned char lvMustInit : 1; // must be initialized diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9bd689d..684d374 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -11936,7 +11936,6 @@ void Compiler::gtDispLocal(GenTreeLclVarCommon* tree, IndentStack* indentStack) printf(" "); fieldVarDsc->PrintVarReg(); } - if (fieldVarDsc->lvTracked && fgLocalVarLivenessDone && tree->IsLastUse(index)) { printf(" (last use)"); @@ -11946,7 +11945,8 @@ void Compiler::gtDispLocal(GenTreeLclVarCommon* tree, IndentStack* indentStack) } else // a normal not-promoted lclvar { - if (varDsc->lvTracked && fgLocalVarLivenessDone && ((tree->gtFlags & GTF_VAR_DEATH) != 0)) + if ((varDsc->lvTracked || varDsc->lvTrackedWithoutIndex) && fgLocalVarLivenessDone && + ((tree->gtFlags & GTF_VAR_DEATH) != 0)) { printf(" (last use)"); } diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 46293ce..79e4b3d 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -3564,6 +3564,7 @@ void Compiler::lvaSortByRefCount() // Start by assuming that the variable will be tracked. varDsc->lvTracked = 1; + INDEBUG(varDsc->lvTrackedWithoutIndex = 0); if (varDsc->lvRefCnt(lvaRefCountState) == 0) { diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index d2df196..7e62b68 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + #include "jitpch.h" #include "promotion.h" #include "jitstd/algorithm.h" @@ -126,6 +129,72 @@ inline AccessKindFlags& operator&=(AccessKindFlags& a, AccessKindFlags b) return a = (AccessKindFlags)((uint32_t)a & (uint32_t)b); } +//------------------------------------------------------------------------ +// OverlappingReplacements: +// Find replacements that overlap the specified [offset..offset+size) interval. +// +// Parameters: +// offset - Starting offset of interval +// size - Size of interval +// firstReplacement - [out] The first replacement that overlaps +// endReplacement - [out, optional] One past the last replacement that overlaps +// +// Returns: +// True if any replacement overlaps; otherwise false. +// +bool AggregateInfo::OverlappingReplacements(unsigned offset, + unsigned size, + Replacement** firstReplacement, + Replacement** endReplacement) +{ + size_t firstIndex = Promotion::BinarySearch(Replacements, offset); + if ((ssize_t)firstIndex < 0) + { + firstIndex = ~firstIndex; + if (firstIndex > 0) + { + Replacement& lastRepBefore = Replacements[firstIndex - 1]; + if ((lastRepBefore.Offset + genTypeSize(lastRepBefore.AccessType)) > offset) + { + // Overlap with last entry starting before offs. + firstIndex--; + } + else if (firstIndex >= Replacements.size()) + { + // Starts after last replacement ends. + return false; + } + } + + const Replacement& first = Replacements[firstIndex]; + if (first.Offset >= (offset + size)) + { + // First candidate starts after this ends. + return false; + } + } + + assert((firstIndex < Replacements.size()) && Replacements[firstIndex].Overlaps(offset, size)); + *firstReplacement = &Replacements[firstIndex]; + + if (endReplacement != nullptr) + { + size_t lastIndex = Promotion::BinarySearch(Replacements, offset + size); + if ((ssize_t)lastIndex < 0) + { + lastIndex = ~lastIndex; + } + + // Since we verified above that there is an overlapping replacement + // we know that lastIndex exists and is the next one that does not + // overlap. + assert(lastIndex > 0); + *endReplacement = Replacements.data() + lastIndex; + } + + return true; +} + // Tracks all the accesses into one particular struct local. class LocalUses { @@ -223,9 +292,9 @@ public: // Parameters: // comp - Compiler instance // lclNum - Local num for this struct local - // replacements - [out] Pointer to vector to create and insert replacements into + // aggregateInfo - [out] Pointer to aggregate info to create and insert replacements into. // - void PickPromotions(Compiler* comp, unsigned lclNum, jitstd::vector** replacements) + void PickPromotions(Compiler* comp, unsigned lclNum, AggregateInfo** aggregateInfo) { if (m_accesses.size() <= 0) { @@ -234,7 +303,7 @@ public: JITDUMP("Picking promotions for V%02u\n", lclNum); - assert(*replacements == nullptr); + assert(*aggregateInfo == nullptr); for (size_t i = 0; i < m_accesses.size(); i++) { const Access& access = m_accesses[i]; @@ -261,13 +330,13 @@ public: LclVarDsc* dsc = comp->lvaGetDesc(newLcl); dsc->lvType = access.AccessType; - if (*replacements == nullptr) + if (*aggregateInfo == nullptr) { - *replacements = - new (comp, CMK_Promotion) jitstd::vector(comp->getAllocator(CMK_Promotion)); + *aggregateInfo = new (comp, CMK_Promotion) AggregateInfo(comp->getAllocator(CMK_Promotion), lclNum); } - (*replacements)->push_back(Replacement(access.Offset, access.AccessType, newLcl DEBUGARG(bufp))); + (*aggregateInfo) + ->Replacements.push_back(Replacement(access.Offset, access.AccessType, newLcl DEBUGARG(bufp))); } JITDUMP("\n"); @@ -619,6 +688,401 @@ bool Replacement::Overlaps(unsigned otherStart, unsigned otherSize) const } //------------------------------------------------------------------------ +// IntersectsOrAdjacent: +// Check if this segment intersects or is adjacent to another segment. +// +// Parameters: +// other - The other segment. +// +// Returns: +// True if so. +// +bool StructSegments::Segment::IntersectsOrAdjacent(const Segment& other) const +{ + if (End < other.Start) + { + return false; + } + + if (other.End < Start) + { + return false; + } + + return true; +} + +//------------------------------------------------------------------------ +// Contains: +// Check if this segment contains another segment. +// +// Parameters: +// other - The other segment. +// +// Returns: +// True if so. +// +bool StructSegments::Segment::Contains(const Segment& other) const +{ + return (other.Start >= Start) && (other.End <= End); +} + +//------------------------------------------------------------------------ +// Merge: +// Update this segment to also contain another segment. +// +// Parameters: +// other - The other segment. +// +void StructSegments::Segment::Merge(const Segment& other) +{ + Start = min(Start, other.Start); + End = max(End, other.End); +} + +//------------------------------------------------------------------------ +// Add: +// Add a segment to the data structure. +// +// Parameters: +// segment - The segment to add. +// +void StructSegments::Add(const Segment& segment) +{ + size_t index = Promotion::BinarySearch(m_segments, segment.Start); + + if ((ssize_t)index < 0) + { + index = ~index; + } + + m_segments.insert(m_segments.begin() + index, segment); + size_t endIndex; + for (endIndex = index + 1; endIndex < m_segments.size(); endIndex++) + { + if (!m_segments[index].IntersectsOrAdjacent(m_segments[endIndex])) + { + break; + } + + m_segments[index].Merge(m_segments[endIndex]); + } + + m_segments.erase(m_segments.begin() + index + 1, m_segments.begin() + endIndex); +} + +//------------------------------------------------------------------------ +// Subtract: +// Subtract a segment from the data structure. +// +// Parameters: +// segment - The segment to subtract. +// +void StructSegments::Subtract(const Segment& segment) +{ + size_t index = Promotion::BinarySearch(m_segments, segment.Start); + if ((ssize_t)index < 0) + { + index = ~index; + } + else + { + // Start == segment[index].End, which makes it non-interesting. + index++; + } + + if (index >= m_segments.size()) + { + return; + } + + // Here we know Start < segment[index].End. Do they not intersect at all? + if (m_segments[index].Start >= segment.End) + { + // Does not intersect any segment. + return; + } + + assert(m_segments[index].IntersectsOrAdjacent(segment)); + + if (m_segments[index].Contains(segment)) + { + if (segment.Start > m_segments[index].Start) + { + // New segment (existing.Start, segment.Start) + if (segment.End < m_segments[index].End) + { + m_segments.insert(m_segments.begin() + index, Segment(m_segments[index].Start, segment.Start)); + + // And new segment (segment.End, existing.End) + m_segments[index + 1].Start = segment.End; + return; + } + + m_segments[index].End = segment.Start; + return; + } + if (segment.End < m_segments[index].End) + { + // New segment (segment.End, existing.End) + m_segments[index].Start = segment.End; + return; + } + + // Full segment is being removed + m_segments.erase(m_segments.begin() + index); + return; + } + + if (segment.Start > m_segments[index].Start) + { + m_segments[index].End = segment.Start; + index++; + } + + size_t endIndex = Promotion::BinarySearch(m_segments, segment.End); + if ((ssize_t)endIndex >= 0) + { + m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex + 1); + return; + } + + endIndex = ~endIndex; + if (endIndex == m_segments.size()) + { + m_segments.erase(m_segments.begin() + index, m_segments.end()); + return; + } + + if (segment.End > m_segments[endIndex].Start) + { + m_segments[endIndex].Start = segment.End; + } + + m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex); +} + +//------------------------------------------------------------------------ +// IsEmpty: +// Check if the segment tree is empty. +// +// Returns: +// True if so. +// +bool StructSegments::IsEmpty() +{ + return m_segments.size() == 0; +} + +//------------------------------------------------------------------------ +// IsSingleSegment: +// Check if the segment tree contains only a single segment, and return +// it if so. +// +// Parameters: +// result - [out] The single segment. Only valid if the method returns true. +// +// Returns: +// True if so. +// +bool StructSegments::IsSingleSegment(Segment* result) +{ + if (m_segments.size() == 1) + { + *result = m_segments[0]; + return true; + } + + return false; +} + +//------------------------------------------------------------------------ +// CoveringSegment: +// Compute a segment that covers all contained segments in this segment tree. +// +// Parameters: +// result - [out] The single segment. Only valid if the method returns true. +// +// Returns: +// True if this segment tree was non-empty; otherwise false. +// +bool StructSegments::CoveringSegment(Segment* result) +{ + if (m_segments.size() == 0) + { + return false; + } + + result->Start = m_segments[0].Start; + result->End = m_segments[m_segments.size() - 1].End; + return true; +} + +#ifdef DEBUG +//------------------------------------------------------------------------ +// Check: +// Validate that the data structure is normalized and that it equals a +// specific fixed bit vector. +// +// Parameters: +// vect - The bit vector +// +// Remarks: +// This validates that the internal representation is normalized (i.e. +// all adjacent intervals are merged) and that it contains an index iff +// the specified vector contains that index. +// +void StructSegments::Check(FixedBitVect* vect) +{ + bool first = true; + unsigned last = 0; + for (const Segment& segment : m_segments) + { + assert(first || (last < segment.Start)); + assert(segment.End <= vect->bitVectGetSize()); + + for (unsigned i = last; i < segment.Start; i++) + assert(!vect->bitVectTest(i)); + + for (unsigned i = segment.Start; i < segment.End; i++) + assert(vect->bitVectTest(i)); + + first = false; + last = segment.End; + } + + for (unsigned i = last, size = vect->bitVectGetSize(); i < size; i++) + assert(!vect->bitVectTest(i)); +} + +//------------------------------------------------------------------------ +// Dump: +// Dump a string representation of the segment tree to stdout. +// +void StructSegments::Dump() +{ + if (m_segments.size() == 0) + { + printf(""); + } + else + { + const char* sep = ""; + for (const Segment& segment : m_segments) + { + printf("%s[%03u..%03u)", sep, segment.Start, segment.End); + sep = " "; + } + } +} +#endif + +//------------------------------------------------------------------------ +// SignificantSegments: +// Compute a segment tree containing all significant (non-padding) segments +// for the specified class layout. +// +// Parameters: +// compiler - Compiler instance +// layout - The layout +// bitVectRept - In debug, a bit vector that represents the same segments as the returned segment tree. +// Used for verification purposes. +// +// Returns: +// Segment tree containing all significant parts of the layout. +// +StructSegments Promotion::SignificantSegments(Compiler* compiler, + ClassLayout* layout DEBUGARG(FixedBitVect** bitVectRepr)) +{ + COMP_HANDLE compHnd = compiler->info.compCompHnd; + + bool significantPadding; + if (layout->IsBlockLayout()) + { + significantPadding = true; + JITDUMP(" Block op has significant padding due to block layout\n"); + } + else + { + uint32_t attribs = compHnd->getClassAttribs(layout->GetClassHandle()); + if ((attribs & CORINFO_FLG_INDEXABLE_FIELDS) != 0) + { + significantPadding = true; + JITDUMP(" Block op has significant padding due to indexable fields\n"); + } + else if ((attribs & CORINFO_FLG_DONT_DIG_FIELDS) != 0) + { + significantPadding = true; + JITDUMP(" Block op has significant padding due to CORINFO_FLG_DONT_DIG_FIELDS\n"); + } + else if (((attribs & CORINFO_FLG_CUSTOMLAYOUT) != 0) && ((attribs & CORINFO_FLG_CONTAINS_GC_PTR) == 0)) + { + significantPadding = true; + JITDUMP(" Block op has significant padding due to CUSTOMLAYOUT without GC pointers\n"); + } + else + { + significantPadding = false; + } + } + + StructSegments segments(compiler->getAllocator(CMK_Promotion)); + + // Validate with "obviously correct" but less scalable fixed bit vector implementation. + INDEBUG(FixedBitVect* segmentBitVect = FixedBitVect::bitVectInit(layout->GetSize(), compiler)); + + if (significantPadding) + { + segments.Add(StructSegments::Segment(0, layout->GetSize())); + +#ifdef DEBUG + for (unsigned i = 0; i < layout->GetSize(); i++) + segmentBitVect->bitVectSet(i); +#endif + } + else + { + unsigned numFields = compHnd->getClassNumInstanceFields(layout->GetClassHandle()); + for (unsigned i = 0; i < numFields; i++) + { + CORINFO_FIELD_HANDLE fieldHnd = compHnd->getFieldInClass(layout->GetClassHandle(), (int)i); + unsigned fldOffset = compHnd->getFieldOffset(fieldHnd); + CORINFO_CLASS_HANDLE fieldClassHandle; + CorInfoType corType = compHnd->getFieldType(fieldHnd, &fieldClassHandle); + var_types varType = JITtype2varType(corType); + unsigned size = genTypeSize(varType); + if (size == 0) + { + // TODO-CQ: Recursively handle padding in sub structures + // here. Might be better to introduce a single JIT-EE call + // to query the significant segments -- that would also be + // usable by R2R even outside the version bubble in many + // cases. + size = compHnd->getClassSize(fieldClassHandle); + assert(size != 0); + } + + segments.Add(StructSegments::Segment(fldOffset, fldOffset + size)); +#ifdef DEBUG + for (unsigned i = 0; i < size; i++) + segmentBitVect->bitVectSet(fldOffset + i); +#endif + } + } + +#ifdef DEBUG + if (bitVectRepr != nullptr) + { + *bitVectRepr = segmentBitVect; + } +#endif + + // TODO-TP: Cache this per class layout, we call this for every struct + // operation on a promoted local. + return segments; +} + +//------------------------------------------------------------------------ // CreateWriteBack: // Create IR that writes a replacement local's value back to its struct local: // @@ -743,6 +1207,11 @@ void ReplaceVisitor::LoadStoreAroundCall(GenTreeCall* call, GenTree* user) { unsigned size = argNodeLcl->GetLayout(m_compiler)->GetSize(); WriteBackBefore(&arg.EarlyNodeRef(), argNodeLcl->GetLclNum(), argNodeLcl->GetLclOffs(), size); + + if ((m_aggregates[argNodeLcl->GetLclNum()] != nullptr) && IsPromotedStructLocalDying(argNodeLcl)) + { + argNodeLcl->gtFlags |= GTF_VAR_DEATH; + } } } @@ -758,6 +1227,46 @@ void ReplaceVisitor::LoadStoreAroundCall(GenTreeCall* call, GenTree* user) } //------------------------------------------------------------------------ +// IsPromotedStructLocalDying: +// Check if a promoted struct local is dying at its current position. +// +// Parameters: +// lcl - The local +// +// Returns: +// True if so. +// +// Remarks: +// This effectively translates our precise liveness information for struct +// uses into the liveness information that the rest of the JIT expects. +// +// If the remainder of the struct local is dying, then we expect that this +// entire struct local is now dying, since all field accesses are going to be +// replaced with other locals. The exception is if there is a queued read +// back for any of the fields. +// +bool ReplaceVisitor::IsPromotedStructLocalDying(GenTreeLclVarCommon* lcl) +{ + StructDeaths deaths = m_liveness->GetDeathsForStructLocal(lcl); + if (!deaths.IsRemainderDying()) + { + return false; + } + + AggregateInfo* agg = m_aggregates[lcl->GetLclNum()]; + + for (size_t i = 0; i < agg->Replacements.size(); i++) + { + if (agg->Replacements[i].NeedsReadBack) + { + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------ // ReplaceLocal: // Handle a local that may need to be replaced. // @@ -780,12 +1289,12 @@ void ReplaceVisitor::ReplaceLocal(GenTree** use, GenTree* user) { GenTreeLclVarCommon* lcl = (*use)->AsLclVarCommon(); unsigned lclNum = lcl->GetLclNum(); - if (m_replacements[lclNum] == nullptr) + if (m_aggregates[lclNum] == nullptr) { return; } - jitstd::vector& replacements = *m_replacements[lclNum]; + jitstd::vector& replacements = m_aggregates[lclNum]->Replacements; unsigned offs = lcl->GetLclOffs(); var_types accessType = lcl->TypeGet(); @@ -828,6 +1337,8 @@ void ReplaceVisitor::ReplaceLocal(GenTree** use, GenTree* user) JITDUMP(" ..replaced with promoted lcl V%02u\n", rep.LclNum); *use = m_compiler->gtNewLclvNode(rep.LclNum, accessType); + (*use)->gtFlags |= lcl->gtFlags & GTF_VAR_DEATH; + if ((lcl->gtFlags & GTF_VAR_DEF) != 0) { (*use)->gtFlags |= GTF_VAR_DEF; // TODO-ASG: delete. @@ -904,12 +1415,12 @@ void ReplaceVisitor::StoreBeforeReturn(GenTreeUnOp* ret) // void ReplaceVisitor::WriteBackBefore(GenTree** use, unsigned lcl, unsigned offs, unsigned size) { - if (m_replacements[lcl] == nullptr) + if (m_aggregates[lcl] == nullptr) { return; } - jitstd::vector& replacements = *m_replacements[lcl]; + jitstd::vector& replacements = m_aggregates[lcl]->Replacements; size_t index = Promotion::BinarySearch(replacements, offs); if ((ssize_t)index < 0) @@ -952,12 +1463,12 @@ void ReplaceVisitor::WriteBackBefore(GenTree** use, unsigned lcl, unsigned offs, // void ReplaceVisitor::MarkForReadBack(unsigned lcl, unsigned offs, unsigned size) { - if (m_replacements[lcl] == nullptr) + if (m_aggregates[lcl] == nullptr) { return; } - jitstd::vector& replacements = *m_replacements[lcl]; + jitstd::vector& replacements = m_aggregates[lcl]->Replacements; size_t index = Promotion::BinarySearch(replacements, offs); if ((ssize_t)index < 0) @@ -1025,9 +1536,8 @@ PhaseStatus Promotion::Run() #endif // Pick promotions based on the use information we just collected. - bool anyReplacements = false; - jitstd::vector** replacements = - new (m_compiler, CMK_Promotion) jitstd::vector*[m_compiler->lvaCount]{}; + bool anyReplacements = false; + jitstd::vector aggregates(m_compiler->lvaCount, nullptr, m_compiler->getAllocator(CMK_Promotion)); for (unsigned i = 0; i < numLocals; i++) { LocalUses* uses = localsUse.GetUsesByLocal(i); @@ -1036,20 +1546,48 @@ PhaseStatus Promotion::Run() continue; } - uses->PickPromotions(m_compiler, i, &replacements[i]); + uses->PickPromotions(m_compiler, i, &aggregates[i]); - if (replacements[i] != nullptr) + if (aggregates[i] == nullptr) { - assert(replacements[i]->size() > 0); - anyReplacements = true; + continue; + } + + jitstd::vector& reps = aggregates[i]->Replacements; + + assert(reps.size() > 0); + anyReplacements = true; #ifdef DEBUG - JITDUMP("V%02u promoted with %d replacements\n", i, (int)replacements[i]->size()); - for (const Replacement& rep : *replacements[i]) - { - JITDUMP(" [%03u..%03u) promoted as %s V%02u\n", rep.Offset, rep.Offset + genTypeSize(rep.AccessType), - varTypeName(rep.AccessType), rep.LclNum); - } + JITDUMP("V%02u promoted with %d replacements\n", i, (int)reps.size()); + for (const Replacement& rep : reps) + { + JITDUMP(" [%03u..%03u) promoted as %s V%02u\n", rep.Offset, rep.Offset + genTypeSize(rep.AccessType), + varTypeName(rep.AccessType), rep.LclNum); + } #endif + + JITDUMP("Computing unpromoted remainder for V%02u\n", i); + StructSegments unpromotedParts = SignificantSegments(m_compiler, m_compiler->lvaGetDesc(i)->GetLayout()); + for (size_t i = 0; i < reps.size(); i++) + { + unpromotedParts.Subtract( + StructSegments::Segment(reps[i].Offset, reps[i].Offset + genTypeSize(reps[i].AccessType))); + } + + JITDUMP(" Remainder: "); + DBEXEC(m_compiler->verbose, unpromotedParts.Dump()); + JITDUMP("\n\n"); + + StructSegments::Segment unpromotedSegment; + if (unpromotedParts.CoveringSegment(&unpromotedSegment)) + { + aggregates[i]->UnpromotedMin = unpromotedSegment.Start; + aggregates[i]->UnpromotedMax = unpromotedSegment.End; + assert(unpromotedSegment.Start < unpromotedSegment.End); + } + else + { + // Aggregate is fully promoted, leave UnpromotedMin == UnpromotedMax to indicate this. } } @@ -1058,8 +1596,13 @@ PhaseStatus Promotion::Run() return PhaseStatus::MODIFIED_NOTHING; } + // Compute liveness for the fields and remainders. + PromotionLiveness liveness(m_compiler, aggregates); + liveness.Run(); + + JITDUMP("Making replacements\n\n"); // Make all replacements we decided on. - ReplaceVisitor replacer(this, replacements); + ReplaceVisitor replacer(this, aggregates, &liveness); for (BasicBlock* bb : m_compiler->Blocks()) { for (Statement* stmt : bb->Statements()) @@ -1079,23 +1622,34 @@ PhaseStatus Promotion::Run() for (unsigned i = 0; i < numLocals; i++) { - if (replacements[i] == nullptr) + if (aggregates[i] == nullptr) { continue; } - for (Replacement& rep : *replacements[i]) + for (size_t j = 0; j < aggregates[i]->Replacements.size(); j++) { + Replacement& rep = aggregates[i]->Replacements[j]; assert(!rep.NeedsReadBack || !rep.NeedsWriteBack); if (rep.NeedsReadBack) { - JITDUMP("Reading back replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB ":\n", i, - rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum); + if (liveness.IsReplacementLiveOut(bb, i, (unsigned)j)) + { + JITDUMP("Reading back replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB ":\n", i, + rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum); - GenTree* readBack = CreateReadBack(m_compiler, i, rep); - Statement* stmt = m_compiler->fgNewStmtFromTree(readBack); - DISPSTMT(stmt); - m_compiler->fgInsertStmtNearEnd(bb, stmt); + GenTree* readBack = CreateReadBack(m_compiler, i, rep); + Statement* stmt = m_compiler->fgNewStmtFromTree(readBack); + DISPSTMT(stmt); + m_compiler->fgInsertStmtNearEnd(bb, stmt); + } + else + { + JITDUMP( + "Skipping reading back dead replacement V%02u.[%03u..%03u) -> V%02u near the end of " FMT_BB + "\n", + i, rep.Offset, rep.Offset + genTypeSize(rep.AccessType), rep.LclNum, bb->bbNum); + } rep.NeedsReadBack = false; } @@ -1109,7 +1663,7 @@ PhaseStatus Promotion::Run() Statement* prevStmt = nullptr; for (unsigned lclNum = 0; lclNum < numLocals; lclNum++) { - if (replacements[lclNum] == nullptr) + if (aggregates[lclNum] == nullptr) { continue; } @@ -1117,7 +1671,7 @@ PhaseStatus Promotion::Run() LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); if (dsc->lvIsParam || dsc->lvIsOSRLocal) { - InsertInitialReadBack(lclNum, *replacements[lclNum], &prevStmt); + InsertInitialReadBack(lclNum, aggregates[lclNum]->Replacements, &prevStmt); } else if (dsc->lvSuppressedZeroInit) { @@ -1126,7 +1680,7 @@ PhaseStatus Promotion::Run() // Now that we are promoting some fields that assumption may be // invalidated for those fields, and we may need to insert explicit // zero inits again. - ExplicitlyZeroInitReplacementLocals(lclNum, *replacements[lclNum], &prevStmt); + ExplicitlyZeroInitReplacementLocals(lclNum, aggregates[lclNum]->Replacements, &prevStmt); } } diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index 6c8f71a..5c4e263 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -37,16 +37,87 @@ struct Replacement bool Overlaps(unsigned otherStart, unsigned otherSize) const; }; +// Represents significant segments of a struct operation. +// +// Essentially a segment tree (but not stored as a tree) that supports boolean +// Add/Subtract operations of segments. Used to compute the remainder after +// replacements have been handled as part of a decomposed block operation. +class StructSegments +{ +public: + struct Segment + { + unsigned Start = 0; + unsigned End = 0; + + Segment() + { + } + + Segment(unsigned start, unsigned end) : Start(start), End(end) + { + } + + bool IntersectsOrAdjacent(const Segment& other) const; + bool Contains(const Segment& other) const; + void Merge(const Segment& other); + }; + +private: + jitstd::vector m_segments; + +public: + StructSegments(CompAllocator allocator) : m_segments(allocator) + { + } + + void Add(const Segment& segment); + void Subtract(const Segment& segment); + bool IsEmpty(); + bool IsSingleSegment(Segment* result); + bool CoveringSegment(Segment* result); + +#ifdef DEBUG + void Check(FixedBitVect* vect); + void Dump(); +#endif +}; + +// Represents information about an aggregate that now has replacements in it. +struct AggregateInfo +{ + jitstd::vector Replacements; + unsigned LclNum; + // Min offset in the struct local of the unpromoted part. + unsigned UnpromotedMin = 0; + // Max offset in the struct local of the unpromoted part. + unsigned UnpromotedMax = 0; + + AggregateInfo(CompAllocator alloc, unsigned lclNum) : Replacements(alloc), LclNum(lclNum) + { + } + + bool OverlappingReplacements(unsigned offset, + unsigned size, + Replacement** firstReplacement, + Replacement** endReplacement); +}; + class Promotion { Compiler* m_compiler; friend class LocalUses; friend class LocalsUseVisitor; + friend struct AggregateInfo; + friend class PromotionLiveness; friend class ReplaceVisitor; friend class DecompositionPlan; friend class StructSegments; + static StructSegments SignificantSegments(Compiler* compiler, + ClassLayout* layout DEBUGARG(FixedBitVect** bitVectRepr = nullptr)); + void InsertInitialReadBack(unsigned lclNum, const jitstd::vector& replacements, Statement** prevStmt); void ExplicitlyZeroInitReplacementLocals(unsigned lclNum, const jitstd::vector& replacements, @@ -106,14 +177,77 @@ public: PhaseStatus Run(); }; +// Class to represent liveness information for a struct local's fields and remainder. +class StructDeaths +{ + BitVec m_deaths; + unsigned m_numFields = 0; + + friend class PromotionLiveness; + +private: + StructDeaths(BitVec deaths, unsigned numFields) : m_deaths(deaths), m_numFields(numFields) + { + } + +public: + StructDeaths() : m_deaths(BitVecOps::UninitVal()) + { + } + + bool IsRemainderDying() const; + bool IsReplacementDying(unsigned index) const; +}; + +struct BasicBlockLiveness; + +// Class to compute and track liveness information pertaining promoted structs. +class PromotionLiveness +{ + Compiler* m_compiler; + jitstd::vector& m_aggregates; + BitVecTraits* m_bvTraits = nullptr; + unsigned* m_structLclToTrackedIndex = nullptr; + unsigned m_numVars = 0; + BasicBlockLiveness* m_bbInfo = nullptr; + bool m_hasPossibleBackEdge = false; + BitVec m_liveIn; + BitVec m_ehLiveVars; + JitHashTable, BitVec> m_aggDeaths; + +public: + PromotionLiveness(Compiler* compiler, jitstd::vector& aggregates) + : m_compiler(compiler), m_aggregates(aggregates), m_aggDeaths(compiler->getAllocator(CMK_Promotion)) + { + } + + void Run(); + bool IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacement); + StructDeaths GetDeathsForStructLocal(GenTreeLclVarCommon* use); + +private: + void MarkUseDef(GenTreeLclVarCommon* lcl, BitVec& useSet, BitVec& defSet); + void MarkIndex(unsigned index, bool isUse, bool isDef, BitVec& useSet, BitVec& defSet); + void ComputeUseDefSets(); + void InterBlockLiveness(); + bool PerBlockLiveness(BasicBlock* block); + void AddHandlerLiveVars(BasicBlock* block, BitVec& ehLiveVars); + void FillInLiveness(); + void FillInLiveness(BitVec& life, BitVec volatileVars, GenTreeLclVarCommon* lcl); +#ifdef DEBUG + void DumpVarSet(BitVec set, BitVec allVars); +#endif +}; + class DecompositionStatementList; class DecompositionPlan; class ReplaceVisitor : public GenTreeVisitor { - Promotion* m_prom; - jitstd::vector** m_replacements; - bool m_madeChanges = false; + Promotion* m_prom; + jitstd::vector& m_aggregates; + PromotionLiveness* m_liveness; + bool m_madeChanges = false; public: enum @@ -122,8 +256,8 @@ public: UseExecutionOrder = true, }; - ReplaceVisitor(Promotion* prom, jitstd::vector** replacements) - : GenTreeVisitor(prom->m_compiler), m_prom(prom), m_replacements(replacements) + ReplaceVisitor(Promotion* prom, jitstd::vector& aggregates, PromotionLiveness* liveness) + : GenTreeVisitor(prom->m_compiler), m_prom(prom), m_aggregates(aggregates), m_liveness(liveness) { } @@ -141,6 +275,7 @@ public: private: void LoadStoreAroundCall(GenTreeCall* call, GenTree* user); + bool IsPromotedStructLocalDying(GenTreeLclVarCommon* structLcl); void ReplaceLocal(GenTree** use, GenTree* user); void StoreBeforeReturn(GenTreeUnOp* ret); void WriteBackBefore(GenTree** use, unsigned lcl, unsigned offs, unsigned size); @@ -160,6 +295,9 @@ private: Replacement* srcEndRep, DecompositionStatementList* statements, DecompositionPlan* plan); +#ifdef DEBUG + const char* LastUseString(GenTreeLclVarCommon* lcl, Replacement* rep); +#endif }; #endif diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index 977c7e0..165607b 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + #include "jitpch.h" #include "promotion.h" #include "jitstd/algorithm.h" @@ -32,278 +35,6 @@ public: } }; -// Represents significant segments of a struct operation. -// -// Essentially a segment tree (but not stored as a tree) that supports boolean -// Add/Subtract operations of segments. Used to compute the remainder after -// replacements have been handled as part of a decomposed block operation. -class StructSegments -{ -public: - struct Segment - { - unsigned Start = 0; - unsigned End = 0; - - Segment() - { - } - - Segment(unsigned start, unsigned end) : Start(start), End(end) - { - } - - bool IntersectsInclusive(const Segment& other) const - { - if (End < other.Start) - { - return false; - } - - if (other.End < Start) - { - return false; - } - - return true; - } - - bool Contains(const Segment& other) const - { - return other.Start >= Start && other.End <= End; - } - - void Merge(const Segment& other) - { - Start = min(Start, other.Start); - End = max(End, other.End); - } - }; - -private: - jitstd::vector m_segments; - -public: - StructSegments(CompAllocator allocator) : m_segments(allocator) - { - } - - //------------------------------------------------------------------------ - // Add: - // Add a segment to the data structure. - // - // Parameters: - // segment - The segment to add. - // - void Add(const Segment& segment) - { - size_t index = Promotion::BinarySearch(m_segments, segment.Start); - - if ((ssize_t)index < 0) - { - index = ~index; - } - - m_segments.insert(m_segments.begin() + index, segment); - size_t endIndex; - for (endIndex = index + 1; endIndex < m_segments.size(); endIndex++) - { - if (!m_segments[index].IntersectsInclusive(m_segments[endIndex])) - { - break; - } - - m_segments[index].Merge(m_segments[endIndex]); - } - - m_segments.erase(m_segments.begin() + index + 1, m_segments.begin() + endIndex); - } - - //------------------------------------------------------------------------ - // Subtract: - // Subtract a segment from the data structure. - // - // Parameters: - // segment - The segment to subtract. - // - void Subtract(const Segment& segment) - { - size_t index = Promotion::BinarySearch(m_segments, segment.Start); - if ((ssize_t)index < 0) - { - index = ~index; - } - else - { - // Start == segment[index].End, which makes it non-interesting. - index++; - } - - if (index >= m_segments.size()) - { - return; - } - - // Here we know Start < segment[index].End. Do they not intersect at all? - if (m_segments[index].Start >= segment.End) - { - // Does not intersect any segment. - return; - } - - assert(m_segments[index].IntersectsInclusive(segment)); - - if (m_segments[index].Contains(segment)) - { - if (segment.Start > m_segments[index].Start) - { - // New segment (existing.Start, segment.Start) - if (segment.End < m_segments[index].End) - { - m_segments.insert(m_segments.begin() + index, Segment(m_segments[index].Start, segment.Start)); - - // And new segment (segment.End, existing.End) - m_segments[index + 1].Start = segment.End; - return; - } - - m_segments[index].End = segment.Start; - return; - } - if (segment.End < m_segments[index].End) - { - // New segment (segment.End, existing.End) - m_segments[index].Start = segment.End; - return; - } - - // Full segment is being removed - m_segments.erase(m_segments.begin() + index); - return; - } - - if (segment.Start > m_segments[index].Start) - { - m_segments[index].End = segment.Start; - index++; - } - - size_t endIndex = Promotion::BinarySearch(m_segments, segment.End); - if ((ssize_t)endIndex >= 0) - { - m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex + 1); - return; - } - - endIndex = ~endIndex; - if (endIndex == m_segments.size()) - { - m_segments.erase(m_segments.begin() + index, m_segments.end()); - return; - } - - if (segment.End > m_segments[endIndex].Start) - { - m_segments[endIndex].Start = segment.End; - } - - m_segments.erase(m_segments.begin() + index, m_segments.begin() + endIndex); - } - - //------------------------------------------------------------------------ - // IsEmpty: - // Check if the segment tree is empty. - // - // Returns: - // True if so. - // - bool IsEmpty() - { - return m_segments.size() == 0; - } - - //------------------------------------------------------------------------ - // IsSingleSegment: - // Check if the segment tree contains only a single segment, and return - // it if so. - // - // Parameters: - // result - [out] The single segment. Only valid if the method returns true. - // - // Returns: - // True if so. - // - bool IsSingleSegment(Segment* result) - { - if (m_segments.size() == 1) - { - *result = m_segments[0]; - return true; - } - - return false; - } - -#ifdef DEBUG - //------------------------------------------------------------------------ - // Check: - // Validate that the data structure is normalized and that it equals a - // specific fixed bit vector. - // - // Parameters: - // vect - The bit vector - // - // Remarks: - // This validates that the internal representation is normalized (i.e. - // all adjacent intervals are merged) and that it contains an index iff - // the specified vector contains that index. - // - void Check(FixedBitVect* vect) - { - bool first = true; - unsigned last = 0; - for (const Segment& segment : m_segments) - { - assert(first || (last < segment.Start)); - assert(segment.End <= vect->bitVectGetSize()); - - for (unsigned i = last; i < segment.Start; i++) - assert(!vect->bitVectTest(i)); - - for (unsigned i = segment.Start; i < segment.End; i++) - assert(vect->bitVectTest(i)); - - first = false; - last = segment.End; - } - - for (unsigned i = last, size = vect->bitVectGetSize(); i < size; i++) - assert(!vect->bitVectTest(i)); - } - - //------------------------------------------------------------------------ - // Dump: - // Dump a string representation of the segment tree to stdout. - // - void Dump() - { - if (m_segments.size() == 0) - { - printf(""); - } - else - { - const char* sep = ""; - for (const Segment& segment : m_segments) - { - printf("%s[%03u..%03u)", sep, segment.Start, segment.End); - sep = " "; - } - } - } -#endif -}; - // Represents a plan for decomposing a block operation into direct treatment of // replacement fields and the remainder. class DecompositionPlan @@ -318,19 +49,32 @@ class DecompositionPlan var_types Type; }; - Compiler* m_compiler; - ArrayStack m_entries; - GenTree* m_dst; - GenTree* m_src; - bool m_srcInvolvesReplacements; + Compiler* m_compiler; + jitstd::vector& m_aggregates; + PromotionLiveness* m_liveness; + GenTree* m_dst; + GenTree* m_src; + bool m_dstInvolvesReplacements; + bool m_srcInvolvesReplacements; + ArrayStack m_entries; + bool m_hasNonRemainderUseOfStructLocal = false; public: - DecompositionPlan(Compiler* comp, GenTree* dst, GenTree* src, bool srcInvolvesReplacements) + DecompositionPlan(Compiler* comp, + jitstd::vector& aggregates, + PromotionLiveness* liveness, + GenTree* dst, + GenTree* src, + bool dstInvolvesReplacements, + bool srcInvolvesReplacements) : m_compiler(comp) - , m_entries(comp->getAllocator(CMK_Promotion)) + , m_aggregates(aggregates) + , m_liveness(liveness) , m_dst(dst) , m_src(src) + , m_dstInvolvesReplacements(dstInvolvesReplacements) , m_srcInvolvesReplacements(srcInvolvesReplacements) + , m_entries(comp->getAllocator(CMK_Promotion)) { } @@ -429,6 +173,17 @@ public: } //------------------------------------------------------------------------ + // MarkNonRemainderUseOfStructLocal: + // Mark that some of the destination replacements are being handled via a + // readback. This invalidates liveness information for the remainder + // because the struct local will now also be used for the readback. + // + void MarkNonRemainderUseOfStructLocal() + { + m_hasNonRemainderUseOfStructLocal = true; + } + + //------------------------------------------------------------------------ // Finalize: // Create IR to perform the full decomposed struct copy as specified by // the entries that were added to the decomposition plan. Add the @@ -516,84 +271,9 @@ private: { ClassLayout* dstLayout = m_dst->GetLayout(m_compiler); - COMP_HANDLE compHnd = m_compiler->info.compCompHnd; - - bool significantPadding; - if (dstLayout->IsBlockLayout()) - { - significantPadding = true; - JITDUMP(" Block op has significant padding due to block layout\n"); - } - else - { - uint32_t attribs = compHnd->getClassAttribs(dstLayout->GetClassHandle()); - if ((attribs & CORINFO_FLG_INDEXABLE_FIELDS) != 0) - { - significantPadding = true; - JITDUMP(" Block op has significant padding due to indexable fields\n"); - } - else if ((attribs & CORINFO_FLG_DONT_DIG_FIELDS) != 0) - { - significantPadding = true; - JITDUMP(" Block op has significant padding due to CORINFO_FLG_DONT_DIG_FIELDS\n"); - } - else if (((attribs & CORINFO_FLG_CUSTOMLAYOUT) != 0) && ((attribs & CORINFO_FLG_CONTAINS_GC_PTR) == 0)) - { - significantPadding = true; - JITDUMP(" Block op has significant padding due to CUSTOMLAYOUT without GC pointers\n"); - } - else - { - significantPadding = false; - } - } - - StructSegments segments(m_compiler->getAllocator(CMK_Promotion)); - // Validate with "obviously correct" but less scalable fixed bit vector implementation. - INDEBUG(FixedBitVect* segmentBitVect = FixedBitVect::bitVectInit(dstLayout->GetSize(), m_compiler)); - - if (significantPadding) - { - segments.Add(StructSegments::Segment(0, dstLayout->GetSize())); - -#ifdef DEBUG - for (unsigned i = 0; i < dstLayout->GetSize(); i++) - segmentBitVect->bitVectSet(i); -#endif - } - else - { - unsigned numFields = compHnd->getClassNumInstanceFields(dstLayout->GetClassHandle()); - for (unsigned i = 0; i < numFields; i++) - { - CORINFO_FIELD_HANDLE fieldHnd = compHnd->getFieldInClass(dstLayout->GetClassHandle(), (int)i); - unsigned fldOffset = compHnd->getFieldOffset(fieldHnd); - CORINFO_CLASS_HANDLE fieldClassHandle; - CorInfoType corType = compHnd->getFieldType(fieldHnd, &fieldClassHandle); - var_types varType = JITtype2varType(corType); - unsigned size = genTypeSize(varType); - if (size == 0) - { - // TODO-CQ: Recursively handle padding in sub structures - // here. Might be better to introduce a single JIT-EE call - // to query the significant segments -- that would also be - // usable by R2R even outside the version bubble in many - // cases. - size = compHnd->getClassSize(fieldClassHandle); - assert(size != 0); - } - - segments.Add(StructSegments::Segment(fldOffset, fldOffset + size)); -#ifdef DEBUG - for (unsigned i = 0; i < size; i++) - segmentBitVect->bitVectSet(fldOffset + i); -#endif - } - } - - // TODO-TP: Cache above StructSegments per class layout and just clone - // it there before the following subtract operations. + INDEBUG(FixedBitVect * segmentBitVect); + StructSegments segments = Promotion::SignificantSegments(m_compiler, dstLayout DEBUGARG(&segmentBitVect)); for (int i = 0; i < m_entries.Height(); i++) { @@ -653,12 +333,18 @@ private: // the rest of the remainder); or by handling a specific 'hole' as a // primitive. // - RemainderStrategy DetermineRemainderStrategy() + RemainderStrategy DetermineRemainderStrategy(const StructDeaths& dstDeaths) { + if (m_dstInvolvesReplacements && !m_hasNonRemainderUseOfStructLocal && dstDeaths.IsRemainderDying()) + { + JITDUMP(" => Remainder strategy: do nothing (remainder dying)\n"); + return RemainderStrategy(RemainderStrategy::NoRemainder); + } + StructSegments remainder = ComputeRemainder(); if (remainder.IsEmpty()) { - JITDUMP(" => Remainder strategy: do nothing\n"); + JITDUMP(" => Remainder strategy: do nothing (no remainder)\n"); return RemainderStrategy(RemainderStrategy::NoRemainder); } @@ -668,7 +354,6 @@ private: { var_types primitiveType = TYP_UNDEF; unsigned size = segment.End - segment.Start; - // For if ((size == TARGET_POINTER_SIZE) && ((segment.Start % TARGET_POINTER_SIZE) == 0)) { ClassLayout* dstLayout = m_dst->GetLayout(m_compiler); @@ -701,7 +386,7 @@ private: { if (!IsInit() || CanInitPrimitive(primitiveType)) { - JITDUMP(" => Remainder strategy: %s at %03u\n", varTypeName(primitiveType), segment.Start); + JITDUMP(" => Remainder strategy: %s at +%03u\n", varTypeName(primitiveType), segment.Start); return RemainderStrategy(RemainderStrategy::Primitive, segment.Start, primitiveType); } else @@ -724,21 +409,33 @@ private: // void FinalizeInit(DecompositionStatementList* statements) { - uint8_t initPattern = GetInitPattern(); + uint8_t initPattern = GetInitPattern(); + StructDeaths deaths = m_liveness->GetDeathsForStructLocal(m_dst->AsLclVarCommon()); + + AggregateInfo* agg = m_aggregates[m_dst->AsLclVarCommon()->GetLclNum()]; + assert((agg != nullptr) && (agg->Replacements.size() > 0)); + Replacement* firstRep = agg->Replacements.data(); for (int i = 0; i < m_entries.Height(); i++) { const Entry& entry = m_entries.BottomRef(i); assert((entry.ToLclNum != BAD_VAR_NUM) && (entry.ToReplacement != nullptr)); - GenTree* src = m_compiler->gtNewConWithPattern(entry.Type, initPattern); - GenTree* dst = m_compiler->gtNewLclvNode(entry.ToLclNum, entry.Type); - statements->AddStatement(m_compiler->gtNewAssignNode(dst, src)); + assert((entry.ToReplacement >= firstRep) && (entry.ToReplacement < firstRep + agg->Replacements.size())); + size_t replacementIndex = entry.ToReplacement - firstRep; + + if (!deaths.IsReplacementDying((unsigned)replacementIndex)) + { + GenTree* src = m_compiler->gtNewConWithPattern(entry.Type, initPattern); + GenTree* dst = m_compiler->gtNewLclvNode(entry.ToLclNum, entry.Type); + statements->AddStatement(m_compiler->gtNewAssignNode(dst, src)); + } + entry.ToReplacement->NeedsWriteBack = true; entry.ToReplacement->NeedsReadBack = false; } - RemainderStrategy remainderStrategy = DetermineRemainderStrategy(); + RemainderStrategy remainderStrategy = DetermineRemainderStrategy(deaths); if (remainderStrategy.Type == RemainderStrategy::FullBlock) { GenTree* asg = m_compiler->gtNewAssignNode(m_dst, m_src); @@ -766,7 +463,13 @@ private: { assert(m_dst->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_BLK) && m_src->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_BLK)); - RemainderStrategy remainderStrategy = DetermineRemainderStrategy(); + StructDeaths dstDeaths; + if (m_dstInvolvesReplacements) + { + dstDeaths = m_liveness->GetDeathsForStructLocal(m_dst->AsLclVarCommon()); + } + + RemainderStrategy remainderStrategy = DetermineRemainderStrategy(dstDeaths); // If the remainder is a full block and is going to incur write barrier // then avoid incurring multiple write barriers for each source @@ -856,7 +559,7 @@ private: { for (int i = 0; i < m_entries.Height(); i++) { - if (!IsHandledByRemainder(m_entries.BottomRef(i), remainderStrategy)) + if (!CanSkipEntry(m_entries.BottomRef(i), dstDeaths, remainderStrategy)) { numAddrUses++; } @@ -879,7 +582,7 @@ private: // See if our first indirection will subsume the null check (usual case). for (int i = 0; i < m_entries.Height(); i++) { - if (IsHandledByRemainder(m_entries.BottomRef(i), remainderStrategy)) + if (CanSkipEntry(m_entries.BottomRef(i), dstDeaths, remainderStrategy)) { continue; } @@ -985,16 +688,37 @@ private: statements->AddStatement(indir); } + StructDeaths srcDeaths; + if (m_srcInvolvesReplacements) + { + srcDeaths = m_liveness->GetDeathsForStructLocal(m_src->AsLclVarCommon()); + } + for (int i = 0; i < m_entries.Height(); i++) { const Entry& entry = m_entries.BottomRef(i); - if (IsHandledByRemainder(entry, remainderStrategy)) + if (entry.ToReplacement != nullptr) { - assert(entry.FromReplacement != nullptr); - JITDUMP(" Skipping dst+%03u <- V%02u (%s); it is up-to-date in its struct local and will be handled " + entry.ToReplacement->NeedsWriteBack = true; + entry.ToReplacement->NeedsReadBack = false; + } + + if (CanSkipEntry(entry, dstDeaths, remainderStrategy)) + { + if (entry.FromReplacement != nullptr) + { + JITDUMP( + " Skipping dst+%03u <- V%02u (%s); it is up-to-date in its struct local and will be handled " "as part of the remainder\n", entry.Offset, entry.FromReplacement->LclNum, entry.FromReplacement->Description); + } + else if (entry.ToReplacement != nullptr) + { + JITDUMP(" Skipping def of V%02u (%s); it is dying", entry.ToReplacement->LclNum, + entry.ToReplacement->Description); + } + continue; } @@ -1023,6 +747,19 @@ private: if (entry.FromLclNum != BAD_VAR_NUM) { src = m_compiler->gtNewLclvNode(entry.FromLclNum, entry.Type); + + if (entry.FromReplacement != nullptr) + { + AggregateInfo* srcAgg = m_aggregates[m_src->AsLclVarCommon()->GetLclNum()]; + Replacement* firstRep = srcAgg->Replacements.data(); + assert((entry.FromReplacement >= firstRep) && + (entry.FromReplacement < (firstRep + srcAgg->Replacements.size()))); + size_t replacementIndex = entry.FromReplacement - firstRep; + if (srcDeaths.IsReplacementDying((unsigned)replacementIndex)) + { + src->gtFlags |= GTF_VAR_DEATH; + } + } } else if (m_src->OperIs(GT_LCL_VAR, GT_LCL_FLD)) { @@ -1040,11 +777,6 @@ private: } statements->AddStatement(m_compiler->gtNewAssignNode(dst, src)); - if (entry.ToReplacement != nullptr) - { - entry.ToReplacement->NeedsWriteBack = true; - entry.ToReplacement->NeedsReadBack = false; - } } if ((remainderStrategy.Type == RemainderStrategy::FullBlock) && !m_srcInvolvesReplacements) @@ -1091,20 +823,41 @@ private: } //------------------------------------------------------------------------ - // IsHandledByRemainder: - // Check if the specified entry is redundant because the remainder would - // handle it anyway. This occurs when we have a source replacement that - // is up-to-date in its struct local and we are going to retain a full - // block operation anyway. + // CanSkipEntry: + // Check if the specified entry can be skipped because it is writing to a + // death replacement or because the remainder would handle it anyway. // // Parameters: // entry - The init/copy entry // remainderStrategy - The strategy we are using for the remainder // - bool IsHandledByRemainder(const Entry& entry, const RemainderStrategy& remainderStrategy) + bool CanSkipEntry(const Entry& entry, const StructDeaths& deaths, const RemainderStrategy& remainderStrategy) { - return (remainderStrategy.Type == RemainderStrategy::FullBlock) && (entry.FromReplacement != nullptr) && - !entry.FromReplacement->NeedsWriteBack && (entry.ToLclNum == BAD_VAR_NUM); + if (entry.ToReplacement != nullptr) + { + // Check if this entry is dying anyway. + assert(m_dstInvolvesReplacements); + + AggregateInfo* agg = m_aggregates[m_dst->AsLclVarCommon()->GetLclNum()]; + assert((agg != nullptr) && (agg->Replacements.size() > 0)); + Replacement* firstRep = agg->Replacements.data(); + assert((entry.ToReplacement >= firstRep) && (entry.ToReplacement < (firstRep + agg->Replacements.size()))); + + size_t replacementIndex = entry.ToReplacement - firstRep; + if (deaths.IsReplacementDying((unsigned)replacementIndex)) + { + return true; + } + } + + if (entry.FromReplacement != nullptr) + { + // Check if the remainder is going to handle it. + return (remainderStrategy.Type == RemainderStrategy::FullBlock) && !entry.FromReplacement->NeedsWriteBack && + (entry.ToLclNum == BAD_VAR_NUM); + } + + return false; } //------------------------------------------------------------------------ @@ -1254,6 +1007,8 @@ void ReplaceVisitor::HandleAssignment(GenTree** use, GenTree* user) if (!dstInvolvesReplacements && !srcInvolvesReplacements) { + // TODO-CQ: If the destination is an aggregate we can still use liveness + // information for the remainder to DCE this. return; } @@ -1264,6 +1019,9 @@ void ReplaceVisitor::HandleAssignment(GenTree** use, GenTree* user) DecompositionStatementList result; EliminateCommasInBlockOp(asg, &result); + DecompositionPlan plan(m_compiler, m_aggregates, m_liveness, dst, src, dstInvolvesReplacements, + srcInvolvesReplacements); + if (dstInvolvesReplacements) { unsigned dstLclOffs = dstLcl->GetLclOffs(); @@ -1283,6 +1041,7 @@ void ReplaceVisitor::HandleAssignment(GenTree** use, GenTree* user) dstFirstRep->NeedsWriteBack = false; } + plan.MarkNonRemainderUseOfStructLocal(); dstFirstRep->NeedsReadBack = true; dstFirstRep++; } @@ -1301,6 +1060,7 @@ void ReplaceVisitor::HandleAssignment(GenTree** use, GenTree* user) dstLastRep->NeedsWriteBack = false; } + plan.MarkNonRemainderUseOfStructLocal(); dstLastRep->NeedsReadBack = true; dstEndRep--; } @@ -1345,8 +1105,6 @@ void ReplaceVisitor::HandleAssignment(GenTree** use, GenTree* user) } } - DecompositionPlan plan(m_compiler, dst, src, srcInvolvesReplacements); - if (src->IsConstInitVal()) { InitFields(dst->AsLclVarCommon(), dstFirstRep, dstEndRep, &plan); @@ -1395,61 +1153,15 @@ bool ReplaceVisitor::OverlappingReplacements(GenTreeLclVarCommon* lcl, Replacement** firstReplacement, Replacement** endReplacement) { - if (m_replacements[lcl->GetLclNum()] == nullptr) + AggregateInfo* agg = m_aggregates[lcl->GetLclNum()]; + if (agg == nullptr) { return false; } - jitstd::vector& replacements = *m_replacements[lcl->GetLclNum()]; - - unsigned offs = lcl->GetLclOffs(); - unsigned size = lcl->GetLayout(m_compiler)->GetSize(); - size_t firstIndex = Promotion::BinarySearch(replacements, offs); - if ((ssize_t)firstIndex < 0) - { - firstIndex = ~firstIndex; - if (firstIndex > 0) - { - Replacement& lastRepBefore = replacements[firstIndex - 1]; - if ((lastRepBefore.Offset + genTypeSize(lastRepBefore.AccessType)) > offs) - { - // Overlap with last entry starting before offs. - firstIndex--; - } - else if (firstIndex >= replacements.size()) - { - // Starts after last replacement ends. - return false; - } - } - - const Replacement& first = replacements[firstIndex]; - if (first.Offset >= (offs + size)) - { - // First candidate starts after this ends. - return false; - } - } - - assert((firstIndex < replacements.size()) && replacements[firstIndex].Overlaps(offs, size)); - *firstReplacement = &replacements[firstIndex]; - - if (endReplacement != nullptr) - { - size_t lastIndex = Promotion::BinarySearch(replacements, offs + size); - if ((ssize_t)lastIndex < 0) - { - lastIndex = ~lastIndex; - } - - // Since we verified above that there is an overlapping replacement - // we know that lastIndex exists and is the next one that does not - // overlap. - assert(lastIndex > 0); - *endReplacement = replacements.data() + lastIndex; - } - - return true; + unsigned offs = lcl->GetLclOffs(); + unsigned size = lcl->GetLayout(m_compiler)->GetSize(); + return agg->OverlappingReplacements(offs, size, firstReplacement, endReplacement); } //------------------------------------------------------------------------ @@ -1542,6 +1254,7 @@ void ReplaceVisitor::InitFields(GenTreeLclVarCommon* dst, // We will need to read this one back after initing the struct. rep->NeedsWriteBack = false; rep->NeedsReadBack = true; + plan->MarkNonRemainderUseOfStructLocal(); continue; } @@ -1550,6 +1263,37 @@ void ReplaceVisitor::InitFields(GenTreeLclVarCommon* dst, } } +#ifdef DEBUG +//------------------------------------------------------------------------ +// LastUseString: +// Return a string indicating whether a replacement is a last use, for +// JITDUMP purposes. +// +// Parameters: +// lcl - A struct local +// rep - A replacement of that struct local +// +// Returns: +// " (last use)" if it is, and otherwise "". +// +const char* ReplaceVisitor::LastUseString(GenTreeLclVarCommon* lcl, Replacement* rep) +{ + StructDeaths deaths = m_liveness->GetDeathsForStructLocal(lcl); + AggregateInfo* agg = m_aggregates[lcl->GetLclNum()]; + assert(agg != nullptr); + Replacement* firstRep = agg->Replacements.data(); + assert((rep >= firstRep) && (rep < firstRep + agg->Replacements.size())); + + size_t replacementIndex = rep - firstRep; + if (deaths.IsReplacementDying((unsigned)replacementIndex)) + { + return " (last use)"; + } + + return ""; +} +#endif + //------------------------------------------------------------------------ // CopyBetweenFields: // Copy between two struct locals that may involve replacements. @@ -1607,7 +1351,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, // Write it directly to the destination struct local. unsigned offs = srcRep->Offset - srcBaseOffs; plan->CopyFromReplacement(srcRep, offs); - JITDUMP(" dst+%03u <- V%02u (%s)\n", offs, srcRep->LclNum, srcRep->Description); + JITDUMP(" dst+%03u <- V%02u (%s)%s\n", offs, srcRep->LclNum, srcRep->Description, + LastUseString(srcLcl, srcRep)); srcRep++; continue; } @@ -1618,7 +1363,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, // Read it directly from the source struct local. unsigned offs = dstRep->Offset - dstBaseOffs; plan->CopyToReplacement(dstRep, offs); - JITDUMP(" V%02u (%s) <- src+%03u\n", dstRep->LclNum, dstRep->Description, offs); + JITDUMP(" V%02u (%s)%s <- src+%03u\n", dstRep->LclNum, dstRep->Description, + LastUseString(dstLcl, dstRep), offs); dstRep++; continue; } @@ -1629,8 +1375,9 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, (dstRep->AccessType == srcRep->AccessType)) { plan->CopyBetweenReplacements(dstRep, srcRep, dstRep->Offset - dstBaseOffs); - JITDUMP(" V%02u (%s) <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description, srcRep->LclNum, - srcRep->Description); + JITDUMP(" V%02u (%s)%s <- V%02u (%s)%s\n", dstRep->LclNum, dstRep->Description, + LastUseString(dstLcl, dstRep), srcRep->LclNum, srcRep->Description, + LastUseString(srcLcl, srcRep)); dstRep++; srcRep++; @@ -1641,8 +1388,9 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, // will handle the destination replacement in a future // iteration of the loop. statements->AddStatement(Promotion::CreateWriteBack(m_compiler, srcLcl->GetLclNum(), *srcRep)); - JITDUMP(" Partial overlap of V%02u (%s) <- V%02u (%s). Will read source back before copy\n", - dstRep->LclNum, dstRep->Description, srcRep->LclNum, srcRep->Description); + JITDUMP(" Partial overlap of V%02u (%s)%s <- V%02u (%s)%s. Will read source back before copy\n", + dstRep->LclNum, dstRep->Description, LastUseString(dstLcl, dstRep), srcRep->LclNum, + srcRep->Description, LastUseString(srcLcl, srcRep)); srcRep++; continue; } @@ -1662,7 +1410,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, if (dsc->lvType == dstRep->AccessType) { plan->CopyBetweenReplacements(dstRep, fieldLcl, offs); - JITDUMP(" V%02u (%s) <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description, dsc->lvReason); + JITDUMP(" V%02u (%s)%s <- V%02u (%s)\n", dstRep->LclNum, dstRep->Description, + LastUseString(dstLcl, dstRep), dsc->lvReason); dstRep++; continue; } @@ -1674,7 +1423,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, // directly to the destination's struct local and mark the // overlapping fields as needing read back to avoid this DNER. plan->CopyToReplacement(dstRep, offs); - JITDUMP(" V%02u (%s) <- src+%03u\n", dstRep->LclNum, dstRep->Description, offs); + JITDUMP(" V%02u (%s)%s <- src+%03u\n", dstRep->LclNum, dstRep->Description, LastUseString(dstLcl, dstRep), + offs); dstRep++; } else @@ -1692,8 +1442,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, if (dsc->lvType == srcRep->AccessType) { plan->CopyBetweenReplacements(fieldLcl, srcRep, offs); - JITDUMP(" V%02u (%s) <- V%02u (%s)\n", fieldLcl, dsc->lvReason, srcRep->LclNum, - srcRep->Description); + JITDUMP(" V%02u (%s) <- V%02u (%s)%s\n", fieldLcl, dsc->lvReason, srcRep->LclNum, + srcRep->Description, LastUseString(srcLcl, srcRep)); srcRep++; continue; } @@ -1701,7 +1451,8 @@ void ReplaceVisitor::CopyBetweenFields(GenTree* dst, } plan->CopyFromReplacement(srcRep, offs); - JITDUMP(" dst+%03u <- V%02u (%s)\n", offs, srcRep->LclNum, srcRep->Description); + JITDUMP(" dst+%03u <- V%02u (%s)%s\n", offs, srcRep->LclNum, srcRep->Description, + LastUseString(srcLcl, srcRep)); srcRep++; } } diff --git a/src/coreclr/jit/promotionliveness.cpp b/src/coreclr/jit/promotionliveness.cpp new file mode 100644 index 0000000..c936478 --- /dev/null +++ b/src/coreclr/jit/promotionliveness.cpp @@ -0,0 +1,869 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "jitpch.h" +#include "promotion.h" + +struct BasicBlockLiveness +{ + // Variables used before a full definition. + BitVec VarUse; + // Variables fully defined before a use. + // Note that this differs from our normal liveness: partial definitions are + // NOT marked but they are also not considered uses. + BitVec VarDef; + // Variables live-in to this basic block. + BitVec LiveIn; + // Variables live-out of this basic block. + BitVec LiveOut; +}; + +//------------------------------------------------------------------------ +// Run: +// Compute liveness information pertaining the promoted structs. +// +// Remarks: +// For each promoted aggregate we compute the liveness for its remainder and +// all of its fields. Unlike regular liveness we currently do not do any DCE +// here and so only do the dataflow computation once. +// +// The liveness information is written into the IR using the normal +// GTF_VAR_DEATH flag. Note that the semantics of GTF_VAR_DEATH differs from +// the rest of the JIT for a short while between the liveness is computed and +// the replacement phase has run: in particular, after this liveness pass you +// may see a node like: +// +// LCL_FLD int V16 tmp9 [+8] (last use) +// +// that indicates that this particular field (or the remainder if it wasn't +// promoted) is dying, not that V16 itself is dying. After replacement has +// run the semantics align with the rest of the JIT: in the promoted case V16 +// [+8] would be replaced by its promoted field local, and in the remainder +// case all non-remainder uses of V16 would also be. +// +// There is one catch which is struct uses of the local. These can indicate +// deaths of multiple fields and also the remainder, so this information is +// stored on the side. PromotionLiveness::GetDeathsForStructLocal is used to +// query this information. +// +// The liveness information is used by decomposition to avoid creating dead +// stores, and also to mark the replacement field uses/defs with proper +// up-to-date liveness information to be used by future phases (forward sub +// and morph, as of writing this). It is also used to avoid creating +// unnecessary read-backs; this is mostly just a TP optimization as future +// liveness passes would be expected to DCE these anyway. +// +// Avoiding the creation of dead stores to the remainder is especially +// important as these otherwise would often end up looking like partial +// definitions, and the other liveness passes handle partial definitions very +// conservatively and are not able to DCE them. +// +// Unlike the other liveness passes we keep the per-block liveness +// information on the side and we do not update BasicBlock::bbLiveIn et al. +// This relies on downstream phases not requiring/wanting to use per-basic +// block live-in/live-out/var-use/var-def sets. To be able to update these we +// would need to give the new locals "regular" tracked indices (i.e. allocate +// a lvVarIndex). +// +// The indices allocated and used internally within the liveness computation +// are "dense" in the sense that the bit vectors only have indices for +// remainders and the replacement fields introduced by this pass. In other +// words, we allocate 1 + num_fields indices for each promoted struct local). +// +void PromotionLiveness::Run() +{ + m_structLclToTrackedIndex = new (m_compiler, CMK_Promotion) unsigned[m_aggregates.size()]{}; + unsigned trackedIndex = 0; + for (size_t lclNum = 0; lclNum < m_aggregates.size(); lclNum++) + { + AggregateInfo* agg = m_aggregates[lclNum]; + if (agg == nullptr) + { + continue; + } + + m_structLclToTrackedIndex[lclNum] = trackedIndex; + // TODO: We need a scalability limit on these, we cannot always track + // the remainder and all fields. + // Remainder. + trackedIndex++; + // Fields. + trackedIndex += (unsigned)agg->Replacements.size(); + +#ifdef DEBUG + // Mark the struct local (remainder) and fields as tracked for DISPTREE to properly + // show last use information. + m_compiler->lvaGetDesc((unsigned)lclNum)->lvTrackedWithoutIndex = true; + for (size_t i = 0; i < agg->Replacements.size(); i++) + { + m_compiler->lvaGetDesc(agg->Replacements[i].LclNum)->lvTrackedWithoutIndex = true; + } +#endif + } + + m_numVars = trackedIndex; + + m_bvTraits = new (m_compiler, CMK_Promotion) BitVecTraits(m_numVars, m_compiler); + m_bbInfo = m_compiler->fgAllocateTypeForEachBlk(CMK_Promotion); + BitVecOps::AssignNoCopy(m_bvTraits, m_liveIn, BitVecOps::MakeEmpty(m_bvTraits)); + BitVecOps::AssignNoCopy(m_bvTraits, m_ehLiveVars, BitVecOps::MakeEmpty(m_bvTraits)); + + JITDUMP("Computing liveness for %u remainders/fields\n\n", m_numVars); + + ComputeUseDefSets(); + + InterBlockLiveness(); + + FillInLiveness(); +} + +//------------------------------------------------------------------------ +// ComputeUseDefSets: +// Compute the use and def sets for all blocks. +// +void PromotionLiveness::ComputeUseDefSets() +{ + for (BasicBlock* block : m_compiler->Blocks()) + { + BasicBlockLiveness& bb = m_bbInfo[block->bbNum]; + BitVecOps::AssignNoCopy(m_bvTraits, bb.VarUse, BitVecOps::MakeEmpty(m_bvTraits)); + BitVecOps::AssignNoCopy(m_bvTraits, bb.VarDef, BitVecOps::MakeEmpty(m_bvTraits)); + BitVecOps::AssignNoCopy(m_bvTraits, bb.LiveIn, BitVecOps::MakeEmpty(m_bvTraits)); + BitVecOps::AssignNoCopy(m_bvTraits, bb.LiveOut, BitVecOps::MakeEmpty(m_bvTraits)); + + if (m_compiler->compQmarkUsed) + { + for (Statement* stmt : block->Statements()) + { + GenTree* dst; + GenTree* qmark = m_compiler->fgGetTopLevelQmark(stmt->GetRootNode(), &dst); + if (qmark == nullptr) + { + for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList()) + { + MarkUseDef(lcl, bb.VarUse, bb.VarDef); + } + } + else + { + for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList()) + { + // Skip liveness updates/marking for defs; they may be conditionally executed. + if ((lcl->gtFlags & GTF_VAR_DEF) == 0) + { + MarkUseDef(lcl, bb.VarUse, bb.VarDef); + } + } + } + } + } + else + { + for (Statement* stmt : block->Statements()) + { + for (GenTreeLclVarCommon* lcl : stmt->LocalsTreeList()) + { + MarkUseDef(lcl, bb.VarUse, bb.VarDef); + } + } + } + +#ifdef DEBUG + if (m_compiler->verbose) + { + BitVec allVars(BitVecOps::Union(m_bvTraits, bb.VarUse, bb.VarDef)); + printf(FMT_BB " USE(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bb.VarUse)); + DumpVarSet(bb.VarUse, allVars); + printf("\n" FMT_BB " DEF(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bb.VarDef)); + DumpVarSet(bb.VarDef, allVars); + printf("\n\n"); + } +#endif + } +} + +//------------------------------------------------------------------------ +// MarkUseDef: +// Mark use/def information for a single appearence of a local. +// +// Parameters: +// lcl - The local node +// useSet - The use set to mark in. +// defSet - The def set to mark in. +// +void PromotionLiveness::MarkUseDef(GenTreeLclVarCommon* lcl, BitSetShortLongRep& useSet, BitSetShortLongRep& defSet) +{ + AggregateInfo* agg = m_aggregates[lcl->GetLclNum()]; + if (agg == nullptr) + { + return; + } + + jitstd::vector& reps = agg->Replacements; + bool isDef = (lcl->gtFlags & GTF_VAR_DEF) != 0; + bool isUse = !isDef; + + unsigned baseIndex = m_structLclToTrackedIndex[lcl->GetLclNum()]; + var_types accessType = lcl->TypeGet(); + + if (accessType == TYP_STRUCT) + { + if (lcl->OperIs(GT_LCL_ADDR)) + { + // For LCL_ADDR this is a retbuf and we expect it to be a def. We + // don't know the exact size here so we cannot mark anything as + // being fully defined, thus we can just return. + assert(isDef); + return; + } + + if (lcl->OperIsScalarLocal()) + { + // Mark remainder and all fields. + for (size_t i = 0; i <= reps.size(); i++) + { + MarkIndex(baseIndex + (unsigned)i, isUse, isDef, useSet, defSet); + } + } + else + { + unsigned offs = lcl->GetLclOffs(); + unsigned size = lcl->GetLayout(m_compiler)->GetSize(); + size_t index = Promotion::BinarySearch(reps, offs); + + if ((ssize_t)index < 0) + { + index = ~index; + if ((index > 0) && reps[index - 1].Overlaps(offs, size)) + { + index--; + } + } + + while ((index < reps.size()) && (reps[index].Offset < offs + size)) + { + Replacement& rep = reps[index]; + bool isFullFieldDef = + isDef && (offs <= rep.Offset) && (offs + size >= rep.Offset + genTypeSize(rep.AccessType)); + MarkIndex(baseIndex + 1 + (unsigned)index, isUse, isFullFieldDef, useSet, defSet); + index++; + } + + bool isFullDefOfRemainder = isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size)); + // TODO-CQ: We could also try to figure out if a use actually touches the remainder, e.g. in some cases + // a struct use may consist only of promoted fields and does not actually use the remainder. + MarkIndex(baseIndex, isUse, isFullDefOfRemainder, useSet, defSet); + } + } + else + { + unsigned offs = lcl->GetLclOffs(); + size_t index = Promotion::BinarySearch(reps, offs); + if ((ssize_t)index < 0) + { + unsigned size = genTypeSize(accessType); + bool isFullDefOfRemainder = isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size)); + MarkIndex(baseIndex, isUse, isFullDefOfRemainder, useSet, defSet); + } + else + { + // Accessing element. + MarkIndex(baseIndex + 1 + (unsigned)index, isUse, isDef, useSet, defSet); + } + } +} + +//------------------------------------------------------------------------ +// MarkIndex: +// Mark specific bits in use/def bit vectors depending on whether this is a use def. +// +// Parameters: +// index - The index of the bit to set. +// isUse - Whether this is a use +// isDef - Whether this is a def +// useSet - The set of uses +// defSet - The set of defs +// +void PromotionLiveness::MarkIndex(unsigned index, bool isUse, bool isDef, BitVec& useSet, BitVec& defSet) +{ + if (isUse && !BitVecOps::IsMember(m_bvTraits, defSet, index)) + { + BitVecOps::AddElemD(m_bvTraits, useSet, index); + } + + if (isDef) + { + BitVecOps::AddElemD(m_bvTraits, defSet, index); + } +} + +//------------------------------------------------------------------------ +// InterBlockLiveness: +// Compute the fixpoint. +// +void PromotionLiveness::InterBlockLiveness() +{ + bool changed; + do + { + changed = false; + + for (BasicBlock* block = m_compiler->fgLastBB; block != nullptr; block = block->bbPrev) + { + m_hasPossibleBackEdge |= block->bbNext && (block->bbNext->bbNum <= block->bbNum); + changed |= PerBlockLiveness(block); + } + + if (!m_hasPossibleBackEdge) + { + break; + } + } while (changed); + +#ifdef DEBUG + if (m_compiler->verbose) + { + for (BasicBlock* block : m_compiler->Blocks()) + { + BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum]; + BitVec allVars(BitVecOps::Union(m_bvTraits, bbInfo.LiveIn, bbInfo.LiveOut)); + printf(FMT_BB " IN (%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bbInfo.LiveIn)); + DumpVarSet(bbInfo.LiveIn, allVars); + printf("\n" FMT_BB " OUT(%u)=", block->bbNum, BitVecOps::Count(m_bvTraits, bbInfo.LiveOut)); + DumpVarSet(bbInfo.LiveOut, allVars); + printf("\n\n"); + } + } +#endif +} + +//------------------------------------------------------------------------ +// PerBlockLiveness: +// Compute liveness for a single block during a single iteration of the +// fixpoint computation. +// +// Parameters: +// block - The block +// +bool PromotionLiveness::PerBlockLiveness(BasicBlock* block) +{ + // We disable promotion for GT_JMP methods. + assert(!block->endsWithJmpMethod(m_compiler)); + + BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum]; + BitVecOps::ClearD(m_bvTraits, bbInfo.LiveOut); + for (BasicBlock* succ : block->GetAllSuccs(m_compiler)) + { + BitVecOps::UnionD(m_bvTraits, bbInfo.LiveOut, m_bbInfo[succ->bbNum].LiveIn); + m_hasPossibleBackEdge |= succ->bbNum <= block->bbNum; + } + + BitVecOps::LivenessD(m_bvTraits, m_liveIn, bbInfo.VarDef, bbInfo.VarUse, bbInfo.LiveOut); + + if (m_compiler->ehBlockHasExnFlowDsc(block)) + { + BitVecOps::ClearD(m_bvTraits, m_ehLiveVars); + AddHandlerLiveVars(block, m_ehLiveVars); + BitVecOps::UnionD(m_bvTraits, m_liveIn, m_ehLiveVars); + BitVecOps::UnionD(m_bvTraits, bbInfo.LiveOut, m_ehLiveVars); + m_hasPossibleBackEdge = true; + } + + bool liveInChanged = !BitVecOps::Equal(m_bvTraits, bbInfo.LiveIn, m_liveIn); + + if (liveInChanged) + { + BitVecOps::Assign(m_bvTraits, bbInfo.LiveIn, m_liveIn); + } + + return liveInChanged; +} + +//------------------------------------------------------------------------ +// AddHandlerLiveVars: +// Find variables that are live-in to handlers reachable by implicit control +// flow and add them to a specified bit vector. +// +// Parameters: +// block - The block +// ehLiveVars - The bit vector to mark in +// +// Remarks: +// Similar to Compiler::fgGetHandlerLiveVars used by regular liveness. +// +void PromotionLiveness::AddHandlerLiveVars(BasicBlock* block, BitVec& ehLiveVars) +{ + assert(m_compiler->ehBlockHasExnFlowDsc(block)); + EHblkDsc* HBtab = m_compiler->ehGetBlockExnFlowDsc(block); + + do + { + // Either we enter the filter first or the catch/finally + if (HBtab->HasFilter()) + { + BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdFilter->bbNum].LiveIn); +#if defined(FEATURE_EH_FUNCLETS) + // The EH subsystem can trigger a stack walk after the filter + // has returned, but before invoking the handler, and the only + // IP address reported from this method will be the original + // faulting instruction, thus everything in the try body + // must report as live any variables live-out of the filter + // (which is the same as those live-in to the handler) + BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdHndBeg->bbNum].LiveIn); +#endif // FEATURE_EH_FUNCLETS + } + else + { + BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[HBtab->ebdHndBeg->bbNum].LiveIn); + } + + // If we have nested try's edbEnclosing will provide them + assert((HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) || + (HBtab->ebdEnclosingTryIndex > m_compiler->ehGetIndex(HBtab))); + + unsigned outerIndex = HBtab->ebdEnclosingTryIndex; + if (outerIndex == EHblkDsc::NO_ENCLOSING_INDEX) + { + break; + } + HBtab = m_compiler->ehGetDsc(outerIndex); + + } while (true); + + // If this block is within a filter, we also need to report as live + // any vars live into enclosed finally or fault handlers, since the + // filter will run during the first EH pass, and enclosed or enclosing + // handlers will run during the second EH pass. So all these handlers + // are "exception flow" successors of the filter. + // + // Note we are relying on ehBlockHasExnFlowDsc to return true + // for any filter block that we should examine here. + if (block->hasHndIndex()) + { + const unsigned thisHndIndex = block->getHndIndex(); + EHblkDsc* enclosingHBtab = m_compiler->ehGetDsc(thisHndIndex); + + if (enclosingHBtab->InFilterRegionBBRange(block)) + { + assert(enclosingHBtab->HasFilter()); + + // Search the EH table for enclosed regions. + // + // All the enclosed regions will be lower numbered and + // immediately prior to and contiguous with the enclosing + // region in the EH tab. + unsigned index = thisHndIndex; + + while (index > 0) + { + index--; + unsigned enclosingIndex = m_compiler->ehGetEnclosingTryIndex(index); + bool isEnclosed = false; + + // To verify this is an enclosed region, search up + // through the enclosing regions until we find the + // region associated with the filter. + while (enclosingIndex != EHblkDsc::NO_ENCLOSING_INDEX) + { + if (enclosingIndex == thisHndIndex) + { + isEnclosed = true; + break; + } + + enclosingIndex = m_compiler->ehGetEnclosingTryIndex(enclosingIndex); + } + + // If we found an enclosed region, check if the region + // is a try fault or try finally, and if so, add any + // locals live into the enclosed region's handler into this + // block's live-in set. + if (isEnclosed) + { + EHblkDsc* enclosedHBtab = m_compiler->ehGetDsc(index); + + if (enclosedHBtab->HasFinallyOrFaultHandler()) + { + BitVecOps::UnionD(m_bvTraits, ehLiveVars, m_bbInfo[enclosedHBtab->ebdHndBeg->bbNum].LiveIn); + } + } + // Once we run across a non-enclosed region, we can stop searching. + else + { + break; + } + } + } + } +} + +//------------------------------------------------------------------------ +// FillInLiveness: +// Starting with the live-out set for each basic block do a backwards traversal +// marking liveness into the IR. +// +void PromotionLiveness::FillInLiveness() +{ + BitVec life(BitVecOps::MakeEmpty(m_bvTraits)); + BitVec volatileVars(BitVecOps::MakeEmpty(m_bvTraits)); + + for (BasicBlock* block : m_compiler->Blocks()) + { + if (block->firstStmt() == nullptr) + { + continue; + } + + BasicBlockLiveness& bbInfo = m_bbInfo[block->bbNum]; + + BitVecOps::ClearD(m_bvTraits, volatileVars); + + if (m_compiler->ehBlockHasExnFlowDsc(block)) + { + AddHandlerLiveVars(block, volatileVars); + } + + BitVecOps::Assign(m_bvTraits, life, bbInfo.LiveOut); + + Statement* stmt = block->lastStmt(); + + while (true) + { + GenTree* qmark = nullptr; + if (m_compiler->compQmarkUsed) + { + GenTree* dst; + qmark = m_compiler->fgGetTopLevelQmark(stmt->GetRootNode(), &dst); + } + + if (qmark == nullptr) + { + for (GenTree* cur = stmt->GetTreeListEnd(); cur != nullptr; cur = cur->gtPrev) + { + FillInLiveness(life, volatileVars, cur->AsLclVarCommon()); + } + } + else + { + for (GenTree* cur = stmt->GetTreeListEnd(); cur != nullptr; cur = cur->gtPrev) + { + // Skip liveness updates/marking for defs; they may be conditionally executed. + if ((cur->gtFlags & GTF_VAR_DEF) == 0) + { + FillInLiveness(life, volatileVars, cur->AsLclVarCommon()); + } + } + } + + if (stmt == block->firstStmt()) + { + break; + } + + stmt = stmt->GetPrevStmt(); + } + } +} + +//------------------------------------------------------------------------ +// FillInLiveness: +// Fill liveness information into the specified IR node. +// +// Parameters: +// life - The current life set. Will be read and updated depending on 'lcl'. +// volatileVars - Bit vector of variables that are live always. +// lcl - The IR node to process liveness for and to mark with liveness information. +// +void PromotionLiveness::FillInLiveness(BitVec& life, BitVec volatileVars, GenTreeLclVarCommon* lcl) +{ + AggregateInfo* agg = m_aggregates[lcl->GetLclNum()]; + if (agg == nullptr) + { + return; + } + + bool isDef = (lcl->gtFlags & GTF_VAR_DEF) != 0; + bool isUse = !isDef; + + unsigned baseIndex = m_structLclToTrackedIndex[lcl->GetLclNum()]; + var_types accessType = lcl->TypeGet(); + + if (accessType == TYP_STRUCT) + { + // We need an external bit set to represent dying fields/remainder on a struct use. + BitVecTraits aggTraits(1 + (unsigned)agg->Replacements.size(), m_compiler); + BitVec aggDeaths(BitVecOps::MakeEmpty(&aggTraits)); + if (lcl->OperIsScalarLocal()) + { + // Handle remainder and all fields. + for (size_t i = 0; i <= agg->Replacements.size(); i++) + { + unsigned varIndex = baseIndex + (unsigned)i; + if (BitVecOps::IsMember(m_bvTraits, life, varIndex)) + { + if (isDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex)) + { + BitVecOps::RemoveElemD(m_bvTraits, life, varIndex); + } + } + else + { + BitVecOps::AddElemD(&aggTraits, aggDeaths, (unsigned)i); + + if (isUse) + { + BitVecOps::AddElemD(m_bvTraits, life, varIndex); + } + } + } + } + else + { + unsigned offs = lcl->GetLclOffs(); + unsigned size = lcl->GetLayout(m_compiler)->GetSize(); + size_t index = Promotion::BinarySearch(agg->Replacements, offs); + + if ((ssize_t)index < 0) + { + index = ~index; + if ((index > 0) && agg->Replacements[index - 1].Overlaps(offs, size)) + { + index--; + } + } + + // Handle fields. + while ((index < agg->Replacements.size()) && (agg->Replacements[index].Offset < offs + size)) + { + unsigned varIndex = baseIndex + 1 + (unsigned)index; + Replacement& rep = agg->Replacements[index]; + if (BitVecOps::IsMember(m_bvTraits, life, varIndex)) + { + bool isFullFieldDef = + isDef && (offs <= rep.Offset) && (offs + size >= rep.Offset + genTypeSize(rep.AccessType)); + if (isFullFieldDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex)) + { + BitVecOps::RemoveElemD(m_bvTraits, life, varIndex); + } + } + else + { + BitVecOps::AddElemD(&aggTraits, aggDeaths, 1 + (unsigned)index); + + if (isUse) + { + BitVecOps::AddElemD(m_bvTraits, life, varIndex); + } + } + + index++; + } + + // Handle remainder. + if (BitVecOps::IsMember(m_bvTraits, life, baseIndex)) + { + bool isFullDefOfRemainder = + isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size)); + if (isFullDefOfRemainder && !BitVecOps::IsMember(m_bvTraits, volatileVars, baseIndex)) + { + BitVecOps::RemoveElemD(m_bvTraits, life, baseIndex); + } + } + else + { + // TODO-CQ: We could also try to figure out if a use actually touches the remainder, e.g. in some cases + // a struct use may consist only of promoted fields and does not actually use the remainder. + BitVecOps::AddElemD(&aggTraits, aggDeaths, 0); + + if (isUse) + { + BitVecOps::AddElemD(m_bvTraits, life, baseIndex); + } + } + } + + m_aggDeaths.Set(lcl, aggDeaths); + } + else + { + if (lcl->OperIs(GT_LCL_ADDR)) + { + // Retbuf -- these are definitions but we do not know of how much. + // We never mark them as dead and we never treat them as killing anything. + assert(isDef); + return; + } + + unsigned offs = lcl->GetLclOffs(); + size_t index = Promotion::BinarySearch(agg->Replacements, offs); + if ((ssize_t)index < 0) + { + // No replacement found, this is a use of the remainder. + unsigned size = genTypeSize(accessType); + if (BitVecOps::IsMember(m_bvTraits, life, baseIndex)) + { + lcl->gtFlags &= ~GTF_VAR_DEATH; + + bool isFullDefOfRemainder = + isDef && (agg->UnpromotedMin >= offs) && (agg->UnpromotedMax <= (offs + size)); + if (isFullDefOfRemainder && !BitVecOps::IsMember(m_bvTraits, volatileVars, baseIndex)) + { + BitVecOps::RemoveElemD(m_bvTraits, life, baseIndex); + } + } + else + { + lcl->gtFlags |= GTF_VAR_DEATH; + + if (isUse) + { + BitVecOps::AddElemD(m_bvTraits, life, baseIndex); + } + } + } + else + { + // Use of a field. + unsigned varIndex = baseIndex + 1 + (unsigned)index; + + if (BitVecOps::IsMember(m_bvTraits, life, varIndex)) + { + lcl->gtFlags &= ~GTF_VAR_DEATH; + + if (isDef && !BitVecOps::IsMember(m_bvTraits, volatileVars, varIndex)) + { + BitVecOps::RemoveElemD(m_bvTraits, life, varIndex); + } + } + else + { + lcl->gtFlags |= GTF_VAR_DEATH; + + if (isUse) + { + BitVecOps::AddElemD(m_bvTraits, life, varIndex); + } + } + } + } +} + +//------------------------------------------------------------------------ +// IsReplacementLiveOut: +// Check if a replacement field is live at the end of a basic block. +// +// Parameters: +// structLcl - The struct (base) local +// replacementIndex - Index of the replacement +// +// Returns: +// True if the field is in the live-out set. +// +bool PromotionLiveness::IsReplacementLiveOut(BasicBlock* bb, unsigned structLcl, unsigned replacementIndex) +{ + BitVec liveOut = m_bbInfo[bb->bbNum].LiveOut; + unsigned baseIndex = m_structLclToTrackedIndex[structLcl]; + return BitVecOps::IsMember(m_bvTraits, liveOut, baseIndex + 1 + replacementIndex); +} + +//------------------------------------------------------------------------ +// GetDeathsForStructLocal: +// Get a data structure that can be used to query liveness information +// for a specified local node at its position. +// +// Parameters: +// lcl - The node +// +// Returns: +// Liveness information. +// +StructDeaths PromotionLiveness::GetDeathsForStructLocal(GenTreeLclVarCommon* lcl) +{ + assert(lcl->OperIsLocal() && lcl->TypeIs(TYP_STRUCT) && (m_aggregates[lcl->GetLclNum()] != nullptr)); + BitVec aggDeaths; + bool found = m_aggDeaths.Lookup(lcl, &aggDeaths); + assert(found); + + unsigned lclNum = lcl->GetLclNum(); + AggregateInfo* aggInfo = m_aggregates[lclNum]; + return StructDeaths(aggDeaths, (unsigned)aggInfo->Replacements.size()); +} + +//------------------------------------------------------------------------ +// IsRemainderDying: +// Check if the remainder is dying. +// +// Returns: +// True if so. +// +bool StructDeaths::IsRemainderDying() const +{ + BitVecTraits traits(1 + m_numFields, nullptr); + return BitVecOps::IsMember(&traits, m_deaths, 0); +} + +//------------------------------------------------------------------------ +// IsReplacementDying: +// Check if a specific replacement is dying. +// +// Returns: +// True if so. +// +bool StructDeaths::IsReplacementDying(unsigned index) const +{ + BitVecTraits traits(1 + m_numFields, nullptr); + return BitVecOps::IsMember(&traits, m_deaths, 1 + index); +} + +#ifdef DEBUG +//------------------------------------------------------------------------ +// DumpVarSet: +// Dump a var set to jitstdout. +// +// Parameters: +// set - The set to dump +// allVars - Set of all variables to print whitespace for if not in 'set'. +// Used for alignment. +// +void PromotionLiveness::DumpVarSet(BitVec set, BitVec allVars) +{ + printf("{"); + + const char* sep = ""; + for (size_t i = 0; i < m_aggregates.size(); i++) + { + AggregateInfo* agg = m_aggregates[i]; + if (agg == nullptr) + { + continue; + } + + for (size_t j = 0; j <= agg->Replacements.size(); j++) + { + unsigned index = (unsigned)(m_structLclToTrackedIndex[i] + j); + + if (BitVecOps::IsMember(m_bvTraits, set, index)) + { + if (j == 0) + { + printf("%sV%02u(remainder)", sep, (unsigned)i); + } + else + { + const Replacement& rep = agg->Replacements[j - 1]; + printf("%sV%02u.[%03u..%03u)", sep, (unsigned)i, rep.Offset, + rep.Offset + genTypeSize(rep.AccessType)); + } + sep = " "; + } + else if (BitVecOps::IsMember(m_bvTraits, allVars, index)) + { + printf("%s ", sep); + sep = " "; + } + } + } + + printf("}"); +} +#endif