Generic Virtual calls for CoreRT
[platform/upstream/coreclr.git] / src / jit / ssabuilder.cpp
index 5d553c1..3d74234 100644 (file)
@@ -103,12 +103,19 @@ void Compiler::fgResetForSsa()
     {
         lvaTable[i].lvPerSsaData.Reset();
     }
-    lvHeapPerSsaData.Reset();
-    m_heapSsaMap = nullptr;
+    lvMemoryPerSsaData.Reset();
+    for (MemoryKind memoryKind : allMemoryKinds())
+    {
+        m_memorySsaMap[memoryKind] = nullptr;
+    }
+
     for (BasicBlock* blk = fgFirstBB; blk != nullptr; blk = blk->bbNext)
     {
         // Eliminate phis.
-        blk->bbHeapSsaPhiFunc = nullptr;
+        for (MemoryKind memoryKind : allMemoryKinds())
+        {
+            blk->bbMemorySsaPhiFunc[memoryKind] = nullptr;
+        }
         if (blk->bbTreeList != nullptr)
         {
             GenTreePtr last = blk->bbTreeList->gtPrev;
@@ -804,29 +811,48 @@ void SsaBuilder::InsertPhiFunctions(BasicBlock** postOrder, int count)
             }
         }
 
-        // Now make a similar phi definition if the block defines Heap.
-        if (block->bbHeapDef)
+        // Now make a similar phi definition if the block defines memory.
+        if (block->bbMemoryDef != 0)
         {
             // For each block "bbInDomFront" that is in the dominance frontier of "block".
             for (BlkSet::KeyIterator iterBlk = blkIdf->Begin(); !iterBlk.Equal(blkIdf->End()); ++iterBlk)
             {
                 BasicBlock* bbInDomFront = iterBlk.Get();
-                DBG_SSA_JITDUMP("     Considering BB%02u in dom frontier of BB%02u for Heap phis:\n",
+                DBG_SSA_JITDUMP("     Considering BB%02u in dom frontier of BB%02u for Memory phis:\n",
                                 bbInDomFront->bbNum, block->bbNum);
 
-                // Check if Heap is live into block "*iterBlk".
-                if (!bbInDomFront->bbHeapLiveIn)
+                for (MemoryKind memoryKind : allMemoryKinds())
                 {
-                    continue;
-                }
+                    if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+                    {
+                        // Share the PhiFunc with ByrefExposed.
+                        assert(memoryKind > ByrefExposed);
+                        bbInDomFront->bbMemorySsaPhiFunc[memoryKind] = bbInDomFront->bbMemorySsaPhiFunc[ByrefExposed];
+                        continue;
+                    }
 
-                // Check if we've already inserted a phi node.
-                if (bbInDomFront->bbHeapSsaPhiFunc == nullptr)
-                {
-                    // We have a variable i that is defined in block j and live at l, and l belongs to dom frontier of
-                    // j. So insert a phi node at l.
-                    JITDUMP("Inserting phi definition for Heap at start of BB%02u.\n", bbInDomFront->bbNum);
-                    bbInDomFront->bbHeapSsaPhiFunc = BasicBlock::EmptyHeapPhiDef;
+                    // Check if this memoryKind is defined in this block.
+                    if ((block->bbMemoryDef & memoryKindSet(memoryKind)) == 0)
+                    {
+                        continue;
+                    }
+
+                    // Check if memoryKind is live into block "*iterBlk".
+                    if ((bbInDomFront->bbMemoryLiveIn & memoryKindSet(memoryKind)) == 0)
+                    {
+                        continue;
+                    }
+
+                    // Check if we've already inserted a phi node.
+                    if (bbInDomFront->bbMemorySsaPhiFunc[memoryKind] == nullptr)
+                    {
+                        // We have a variable i that is defined in block j and live at l, and l belongs to dom frontier
+                        // of
+                        // j. So insert a phi node at l.
+                        JITDUMP("Inserting phi definition for %s at start of BB%02u.\n", memoryKindNames[memoryKind],
+                                bbInDomFront->bbNum);
+                        bbInDomFront->bbMemorySsaPhiFunc[memoryKind] = BasicBlock::EmptyMemoryPhiDef;
+                    }
                 }
             }
         }
@@ -944,31 +970,63 @@ void SsaBuilder::TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRename
         }
     }
 
-    // Figure out if "tree" may make a new heap state (if we care for this block).
-    if (!block->bbHeapHavoc)
+    // Figure out if "tree" may make a new GC heap state (if we care for this block).
+    if ((block->bbMemoryHavoc & memoryKindSet(GcHeap)) == 0)
     {
         if (tree->OperIsAssignment() || tree->OperIsBlkOp())
         {
             if (m_pCompiler->ehBlockHasExnFlowDsc(block))
             {
                 GenTreeLclVarCommon* lclVarNode;
-                if (!tree->DefinesLocal(m_pCompiler, &lclVarNode))
+
+                bool isLocal            = tree->DefinesLocal(m_pCompiler, &lclVarNode);
+                bool isAddrExposedLocal = isLocal && m_pCompiler->lvaVarAddrExposed(lclVarNode->gtLclNum);
+                bool hasByrefHavoc      = ((block->bbMemoryHavoc & memoryKindSet(ByrefExposed)) != 0);
+                if (!isLocal || (isAddrExposedLocal && !hasByrefHavoc))
                 {
-                    // It *may* define the heap in a non-havoc way.  Make a new SSA # -- associate with this node.
-                    unsigned count = pRenameState->CountForHeapDef();
-                    pRenameState->PushHeap(block, count);
-                    m_pCompiler->GetHeapSsaMap()->Set(tree, count);
-#ifdef DEBUG
-                    if (JitTls::GetCompiler()->verboseSsa)
+                    // It *may* define byref memory in a non-havoc way.  Make a new SSA # -- associate with this node.
+                    unsigned count = pRenameState->CountForMemoryDef();
+                    if (!hasByrefHavoc)
                     {
-                        printf("Node ");
-                        Compiler::printTreeID(tree);
-                        printf(" (in try block) may define heap; ssa # = %d.\n", count);
-                    }
+                        pRenameState->PushMemory(ByrefExposed, block, count);
+                        m_pCompiler->GetMemorySsaMap(ByrefExposed)->Set(tree, count);
+#ifdef DEBUG
+                        if (JitTls::GetCompiler()->verboseSsa)
+                        {
+                            printf("Node ");
+                            Compiler::printTreeID(tree);
+                            printf(" (in try block) may define memory; ssa # = %d.\n", count);
+                        }
 #endif // DEBUG
 
-                    // Now add this SSA # to all phis of the reachable catch blocks.
-                    AddHeapDefToHandlerPhis(block, count);
+                        // Now add this SSA # to all phis of the reachable catch blocks.
+                        AddMemoryDefToHandlerPhis(ByrefExposed, block, count);
+                    }
+
+                    if (!isLocal)
+                    {
+                        // Add a new def for GcHeap as well
+                        if (m_pCompiler->byrefStatesMatchGcHeapStates)
+                        {
+                            // GcHeap and ByrefExposed share the same stacks, SsaMap, and phis
+                            assert(!hasByrefHavoc);
+                            assert(pRenameState->CountForMemoryUse(GcHeap) == count);
+                            assert(*m_pCompiler->GetMemorySsaMap(GcHeap)->LookupPointer(tree) == count);
+                            assert(block->bbMemorySsaPhiFunc[GcHeap] == block->bbMemorySsaPhiFunc[ByrefExposed]);
+                        }
+                        else
+                        {
+                            if (!hasByrefHavoc)
+                            {
+                                // Allocate a distinct defnum for the GC Heap
+                                count = pRenameState->CountForMemoryDef();
+                            }
+
+                            pRenameState->PushMemory(GcHeap, block, count);
+                            m_pCompiler->GetMemorySsaMap(GcHeap)->Set(tree, count);
+                            AddMemoryDefToHandlerPhis(GcHeap, block, count);
+                        }
+                    }
                 }
             }
         }
@@ -1154,7 +1212,7 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne
     }
 }
 
-void SsaBuilder::AddHeapDefToHandlerPhis(BasicBlock* block, unsigned count)
+void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* block, unsigned count)
 {
     if (m_pCompiler->ehBlockHasExnFlowDsc(block))
     {
@@ -1165,39 +1223,60 @@ void SsaBuilder::AddHeapDefToHandlerPhis(BasicBlock* block, unsigned count)
         }
 
         // Otherwise...
-        DBG_SSA_JITDUMP("Definition of Heap/d:%d in block BB%02u has exn handler; adding as phi arg to handlers.\n",
-                        count, block->bbNum);
+        DBG_SSA_JITDUMP("Definition of %s/d:%d in block BB%02u has exn handler; adding as phi arg to handlers.\n",
+                        memoryKindNames[memoryKind], count, block->bbNum);
         EHblkDsc* tryBlk = m_pCompiler->ehGetBlockExnFlowDsc(block);
         while (true)
         {
             BasicBlock* handler = tryBlk->ExFlowBlock();
 
-            // Is Heap live on entry to the handler?
-            if (handler->bbHeapLiveIn)
+            // Is memoryKind live on entry to the handler?
+            if ((handler->bbMemoryLiveIn & memoryKindSet(memoryKind)) != 0)
             {
-                assert(handler->bbHeapSsaPhiFunc != nullptr);
+                assert(handler->bbMemorySsaPhiFunc != nullptr);
+
+                // Add "count" to the phi args of memoryKind.
+                BasicBlock::MemoryPhiArg*& handlerMemoryPhi = handler->bbMemorySsaPhiFunc[memoryKind];
+
+#if DEBUG
+                if (m_pCompiler->byrefStatesMatchGcHeapStates)
+                {
+                    // When sharing phis for GcHeap and ByrefExposed, callers should ask to add phis
+                    // for ByrefExposed only.
+                    assert(memoryKind != GcHeap);
+                    if (memoryKind == ByrefExposed)
+                    {
+                        // The GcHeap and ByrefExposed phi funcs should always be in sync.
+                        assert(handlerMemoryPhi == handler->bbMemorySsaPhiFunc[GcHeap]);
+                    }
+                }
+#endif
 
-                // Add "count" to the phi args of Heap.
-                if (handler->bbHeapSsaPhiFunc == BasicBlock::EmptyHeapPhiDef)
+                if (handlerMemoryPhi == BasicBlock::EmptyMemoryPhiDef)
                 {
-                    handler->bbHeapSsaPhiFunc = new (m_pCompiler) BasicBlock::HeapPhiArg(count);
+                    handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(count);
                 }
                 else
                 {
 #ifdef DEBUG
-                    BasicBlock::HeapPhiArg* curArg = handler->bbHeapSsaPhiFunc;
+                    BasicBlock::MemoryPhiArg* curArg = handler->bbMemorySsaPhiFunc[memoryKind];
                     while (curArg != nullptr)
                     {
                         assert(curArg->GetSsaNum() != count);
                         curArg = curArg->m_nextArg;
                     }
 #endif // DEBUG
-                    handler->bbHeapSsaPhiFunc =
-                        new (m_pCompiler) BasicBlock::HeapPhiArg(count, handler->bbHeapSsaPhiFunc);
+                    handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(count, handlerMemoryPhi);
                 }
 
-                DBG_SSA_JITDUMP("   Added phi arg u:%d for Heap to phi defn in handler block BB%02u.\n", count,
-                                handler->bbNum);
+                DBG_SSA_JITDUMP("   Added phi arg u:%d for %s to phi defn in handler block BB%02u.\n", count,
+                                memoryKindNames[memoryKind], memoryKind, handler->bbNum);
+
+                if ((memoryKind == ByrefExposed) && m_pCompiler->byrefStatesMatchGcHeapStates)
+                {
+                    // Share the phi between GcHeap and ByrefExposed.
+                    handler->bbMemorySsaPhiFunc[GcHeap] = handlerMemoryPhi;
+                }
             }
             unsigned tryInd = tryBlk->ebdEnclosingTryIndex;
             if (tryInd == EHblkDsc::NO_ENCLOSING_INDEX)
@@ -1221,19 +1300,33 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename
 {
     // Walk the statements of the block and rename the tree variables.
 
-    // First handle the incoming Heap state.
-
-    // Is there an Phi definition for heap at the start of this block?
-    if (block->bbHeapSsaPhiFunc != nullptr)
+    // First handle the incoming memory states.
+    for (MemoryKind memoryKind : allMemoryKinds())
     {
-        unsigned count = pRenameState->CountForHeapDef();
-        pRenameState->PushHeap(block, count);
+        if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+        {
+            // ByrefExposed and GcHeap share any phi this block may have,
+            assert(block->bbMemorySsaPhiFunc[memoryKind] == block->bbMemorySsaPhiFunc[ByrefExposed]);
+            // so we will have already allocated a defnum for it if needed.
+            assert(memoryKind > ByrefExposed);
+            assert(pRenameState->CountForMemoryUse(memoryKind) == pRenameState->CountForMemoryUse(ByrefExposed));
+        }
+        else
+        {
+            // Is there an Phi definition for memoryKind at the start of this block?
+            if (block->bbMemorySsaPhiFunc[memoryKind] != nullptr)
+            {
+                unsigned count = pRenameState->CountForMemoryDef();
+                pRenameState->PushMemory(memoryKind, block, count);
 
-        DBG_SSA_JITDUMP("Ssa # for Heap phi on entry to BB%02u is %d.\n", block->bbNum, count);
-    }
+                DBG_SSA_JITDUMP("Ssa # for %s phi on entry to BB%02u is %d.\n", memoryKindNames[memoryKind],
+                                block->bbNum, count);
+            }
+        }
 
-    // Record the "in" Ssa # for Heap.
-    block->bbHeapSsaNumIn = pRenameState->CountForHeapUse();
+        // Record the "in" Ssa # for memoryKind.
+        block->bbMemorySsaNumIn[memoryKind] = pRenameState->CountForMemoryUse(memoryKind);
+    }
 
     // We need to iterate over phi definitions, to give them SSA names, but we need
     // to know which are which, so we don't add phi definitions to handler phi arg lists.
@@ -1253,22 +1346,38 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename
         }
     }
 
-    // Now handle the final heap state.
-
-    // If the block defines Heap, allocate an SSA variable for the final heap state in the block.
-    // (This may be redundant with the last SSA var explicitly created, but there's no harm in that.)
-    if (block->bbHeapDef)
+    // Now handle the final memory states.
+    for (MemoryKind memoryKind : allMemoryKinds())
     {
-        unsigned count = pRenameState->CountForHeapDef();
-        pRenameState->PushHeap(block, count);
-        AddHeapDefToHandlerPhis(block, count);
-    }
+        MemoryKindSet memorySet = memoryKindSet(memoryKind);
+
+        // If the block defines memory, allocate an SSA variable for the final memory state in the block.
+        // (This may be redundant with the last SSA var explicitly created, but there's no harm in that.)
+        if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+        {
+            // We've already allocated the SSA num and propagated it to shared phis, if needed,
+            // when processing ByrefExposed.
+            assert(memoryKind > ByrefExposed);
+            assert(((block->bbMemoryDef & memorySet) != 0) ==
+                   ((block->bbMemoryDef & memoryKindSet(ByrefExposed)) != 0));
+            assert(pRenameState->CountForMemoryUse(memoryKind) == pRenameState->CountForMemoryUse(ByrefExposed));
+        }
+        else
+        {
+            if ((block->bbMemoryDef & memorySet) != 0)
+            {
+                unsigned count = pRenameState->CountForMemoryDef();
+                pRenameState->PushMemory(memoryKind, block, count);
+                AddMemoryDefToHandlerPhis(memoryKind, block, count);
+            }
+        }
 
-    // Record the "out" Ssa" # for Heap.
-    block->bbHeapSsaNumOut = pRenameState->CountForHeapUse();
+        // Record the "out" Ssa" # for memoryKind.
+        block->bbMemorySsaNumOut[memoryKind] = pRenameState->CountForMemoryUse(memoryKind);
 
-    DBG_SSA_JITDUMP("Ssa # for Heap on entry to BB%02u is %d; on exit is %d.\n", block->bbNum, block->bbHeapSsaNumIn,
-                    block->bbHeapSsaNumOut);
+        DBG_SSA_JITDUMP("Ssa # for %s on entry to BB%02u is %d; on exit is %d.\n", memoryKindNames[memoryKind],
+                        block->bbNum, block->bbMemorySsaNumIn[memoryKind], block->bbMemorySsaNumOut[memoryKind]);
+    }
 }
 
 /**
@@ -1328,35 +1437,54 @@ void SsaBuilder::AssignPhiNodeRhsVariables(BasicBlock* block, SsaRenameState* pR
             m_pCompiler->fgSetStmtSeq(stmt);
         }
 
-        // Now handle Heap.
-        if (succ->bbHeapSsaPhiFunc != nullptr)
+        // Now handle memory.
+        for (MemoryKind memoryKind : allMemoryKinds())
         {
-            if (succ->bbHeapSsaPhiFunc == BasicBlock::EmptyHeapPhiDef)
-            {
-                succ->bbHeapSsaPhiFunc = new (m_pCompiler) BasicBlock::HeapPhiArg(block->bbHeapSsaNumOut);
-            }
-            else
+            BasicBlock::MemoryPhiArg*& succMemoryPhi = succ->bbMemorySsaPhiFunc[memoryKind];
+            if (succMemoryPhi != nullptr)
             {
-                BasicBlock::HeapPhiArg* curArg = succ->bbHeapSsaPhiFunc;
-                unsigned                ssaNum = block->bbHeapSsaNumOut;
-                bool                    found  = false;
-                // This is a quadratic algorithm.  We might need to consider some switch over to a hash table
-                // representation for the arguments of a phi node, to make this linear.
-                while (curArg != nullptr)
+                if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
                 {
-                    if (curArg->m_ssaNum == ssaNum)
-                    {
-                        found = true;
-                        break;
-                    }
-                    curArg = curArg->m_nextArg;
+                    // We've already propagated the "out" number to the phi shared with ByrefExposed,
+                    // but still need to update bbMemorySsaPhiFunc to be in sync between GcHeap and ByrefExposed.
+                    assert(memoryKind > ByrefExposed);
+                    assert(block->bbMemorySsaNumOut[memoryKind] == block->bbMemorySsaNumOut[ByrefExposed]);
+                    assert((succ->bbMemorySsaPhiFunc[ByrefExposed] == succMemoryPhi) ||
+                           (succ->bbMemorySsaPhiFunc[ByrefExposed]->m_nextArg ==
+                            (succMemoryPhi == BasicBlock::EmptyMemoryPhiDef ? nullptr : succMemoryPhi)));
+                    succMemoryPhi = succ->bbMemorySsaPhiFunc[ByrefExposed];
+
+                    continue;
+                }
+
+                if (succMemoryPhi == BasicBlock::EmptyMemoryPhiDef)
+                {
+                    succMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(block->bbMemorySsaNumOut[memoryKind]);
                 }
-                if (!found)
+                else
                 {
-                    succ->bbHeapSsaPhiFunc = new (m_pCompiler) BasicBlock::HeapPhiArg(ssaNum, succ->bbHeapSsaPhiFunc);
+                    BasicBlock::MemoryPhiArg* curArg = succMemoryPhi;
+                    unsigned                  ssaNum = block->bbMemorySsaNumOut[memoryKind];
+                    bool                      found  = false;
+                    // This is a quadratic algorithm.  We might need to consider some switch over to a hash table
+                    // representation for the arguments of a phi node, to make this linear.
+                    while (curArg != nullptr)
+                    {
+                        if (curArg->m_ssaNum == ssaNum)
+                        {
+                            found = true;
+                            break;
+                        }
+                        curArg = curArg->m_nextArg;
+                    }
+                    if (!found)
+                    {
+                        succMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(ssaNum, succMemoryPhi);
+                    }
                 }
+                DBG_SSA_JITDUMP("  Added phi arg for %s u:%d from BB%02u in BB%02u.\n", memoryKindNames[memoryKind],
+                                block->bbMemorySsaNumOut[memoryKind], block->bbNum, succ->bbNum);
             }
-            DBG_SSA_JITDUMP("  Added phi arg for Heap from BB%02u in BB%02u.\n", block->bbNum, succ->bbNum);
         }
 
         // If "succ" is the first block of a try block (and "block" is not also in that try block)
@@ -1462,26 +1590,44 @@ void SsaBuilder::AssignPhiNodeRhsVariables(BasicBlock* block, SsaRenameState* pR
                     }
                 }
 
-                // Now handle Heap.
-                if (handlerStart->bbHeapSsaPhiFunc != nullptr)
+                // Now handle memory.
+                for (MemoryKind memoryKind : allMemoryKinds())
                 {
-                    if (handlerStart->bbHeapSsaPhiFunc == BasicBlock::EmptyHeapPhiDef)
-                    {
-                        handlerStart->bbHeapSsaPhiFunc =
-                            new (m_pCompiler) BasicBlock::HeapPhiArg(block->bbHeapSsaNumOut);
-                    }
-                    else
+                    BasicBlock::MemoryPhiArg*& handlerMemoryPhi = handlerStart->bbMemorySsaPhiFunc[memoryKind];
+                    if (handlerMemoryPhi != nullptr)
                     {
-                        // This path has a potential to introduce redundant phi args, due to multiple
-                        // preds of the same try-begin block having the same live-out heap def, and/or
-                        // due to nested try-begins each having preds with the same live-out heap def.
-                        // Avoid doing quadratic processing on handler phis, and instead live with the
-                        // occasional redundancy.
-                        handlerStart->bbHeapSsaPhiFunc = new (m_pCompiler)
-                            BasicBlock::HeapPhiArg(block->bbHeapSsaNumOut, handlerStart->bbHeapSsaPhiFunc);
+                        if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+                        {
+                            // We've already added the arg to the phi shared with ByrefExposed if needed,
+                            // but still need to update bbMemorySsaPhiFunc to stay in sync.
+                            assert(memoryKind > ByrefExposed);
+                            assert(block->bbMemorySsaNumOut[memoryKind] == block->bbMemorySsaNumOut[ByrefExposed]);
+                            assert(handlerStart->bbMemorySsaPhiFunc[ByrefExposed]->m_ssaNum ==
+                                   block->bbMemorySsaNumOut[memoryKind]);
+                            handlerMemoryPhi = handlerStart->bbMemorySsaPhiFunc[ByrefExposed];
+
+                            continue;
+                        }
+
+                        if (handlerMemoryPhi == BasicBlock::EmptyMemoryPhiDef)
+                        {
+                            handlerMemoryPhi =
+                                new (m_pCompiler) BasicBlock::MemoryPhiArg(block->bbMemorySsaNumOut[memoryKind]);
+                        }
+                        else
+                        {
+                            // This path has a potential to introduce redundant phi args, due to multiple
+                            // preds of the same try-begin block having the same live-out memory def, and/or
+                            // due to nested try-begins each having preds with the same live-out memory def.
+                            // Avoid doing quadratic processing on handler phis, and instead live with the
+                            // occasional redundancy.
+                            handlerMemoryPhi = new (m_pCompiler)
+                                BasicBlock::MemoryPhiArg(block->bbMemorySsaNumOut[memoryKind], handlerMemoryPhi);
+                        }
+                        DBG_SSA_JITDUMP("  Added phi arg for %s u:%d from BB%02u in BB%02u.\n",
+                                        memoryKindNames[memoryKind], block->bbMemorySsaNumOut[memoryKind], block->bbNum,
+                                        handlerStart->bbNum);
                     }
-                    DBG_SSA_JITDUMP("  Added phi arg for Heap from BB%02u in BB%02u.\n", block->bbNum,
-                                    handlerStart->bbNum);
                 }
 
                 tryInd = succTry->ebdEnclosingTryIndex;
@@ -1502,8 +1648,17 @@ void SsaBuilder::BlockPopStacks(BasicBlock* block, SsaRenameState* pRenameState)
     // Pop the names given to the non-phi nodes.
     pRenameState->PopBlockStacks(block);
 
-    // And for Heap.
-    pRenameState->PopBlockHeapStack(block);
+    // And for memory.
+    for (MemoryKind memoryKind : allMemoryKinds())
+    {
+        if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+        {
+            // GcHeap and ByrefExposed share a rename stack, so don't try
+            // to pop it a second time.
+            continue;
+        }
+        pRenameState->PopBlockMemoryStack(memoryKind, block);
+    }
 }
 
 /**
@@ -1552,20 +1707,32 @@ void SsaBuilder::RenameVariables(BlkToBlkSetMap* domTree, SsaRenameState* pRenam
             pRenameState->Push(nullptr, i, count);
         }
     }
-    // In ValueNum we'd assume un-inited heap gets FIRST_SSA_NUM.
-    // The heap is a parameter.  Use FIRST_SSA_NUM as first SSA name.
-    unsigned initHeapCount = pRenameState->CountForHeapDef();
-    assert(initHeapCount == SsaConfig::FIRST_SSA_NUM);
-    pRenameState->PushHeap(m_pCompiler->fgFirstBB, initHeapCount);
-
-    // Initialize the heap ssa numbers for unreachable blocks. ValueNum expects
-    // heap ssa numbers to have some intitial value.
+
+    // In ValueNum we'd assume un-inited memory gets FIRST_SSA_NUM.
+    // The memory is a parameter.  Use FIRST_SSA_NUM as first SSA name.
+    unsigned initMemoryCount = pRenameState->CountForMemoryDef();
+    assert(initMemoryCount == SsaConfig::FIRST_SSA_NUM);
+    for (MemoryKind memoryKind : allMemoryKinds())
+    {
+        if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates)
+        {
+            // GcHeap shares its stack with ByrefExposed; don't re-push.
+            continue;
+        }
+        pRenameState->PushMemory(memoryKind, m_pCompiler->fgFirstBB, initMemoryCount);
+    }
+
+    // Initialize the memory ssa numbers for unreachable blocks. ValueNum expects
+    // memory ssa numbers to have some intitial value.
     for (BasicBlock* block = m_pCompiler->fgFirstBB; block; block = block->bbNext)
     {
         if (block->bbIDom == nullptr)
         {
-            block->bbHeapSsaNumIn  = initHeapCount;
-            block->bbHeapSsaNumOut = initHeapCount;
+            for (MemoryKind memoryKind : allMemoryKinds())
+            {
+                block->bbMemorySsaNumIn[memoryKind]  = initMemoryCount;
+                block->bbMemorySsaNumOut[memoryKind] = initMemoryCount;
+            }
         }
     }
 
@@ -1624,8 +1791,8 @@ void SsaBuilder::RenameVariables(BlkToBlkSetMap* domTree, SsaRenameState* pRenam
         }
     }
 
-    // Remember the number of Heap SSA names.
-    m_pCompiler->lvHeapNumSsaNames = pRenameState->HeapCount();
+    // Remember the number of memory SSA names.
+    m_pCompiler->lvMemoryNumSsaNames = pRenameState->MemoryCount();
 }
 
 #ifdef DEBUG
@@ -1732,7 +1899,7 @@ void SsaBuilder::Build()
 
     // Rename local variables and collect UD information for each ssa var.
     SsaRenameState* pRenameState = new (jitstd::utility::allocate<SsaRenameState>(m_allocator), jitstd::placement_t())
-        SsaRenameState(m_allocator, m_pCompiler->lvaCount);
+        SsaRenameState(m_allocator, m_pCompiler->lvaCount, m_pCompiler->byrefStatesMatchGcHeapStates);
     RenameVariables(domTree, pRenameState);
     EndPhase(PHASE_BUILD_SSA_RENAME);