DacEnumerableHashValue m_iHashValue; // The hash value associated with the entry
};
+
+ // End sentinel logic
+ // End sentinels indicate the end of the linked list of VolatileEntrys of each bucket
+ // As the list can be mutated while readers are reading, we use a specific end sentinel
+ // per list to identify the end of the list instead of NULL. We take advantage of the
+ // concept that VolatileEntry values are aligned, and distinguish an end sentinel
+ // from a normal pointer by means of the low bit, and we use an incrementing value
+ // for the rest of the end sentinel so that we can detect when we finish walking
+ // a linked list if we found ourselves walking to the end of the expected list,
+ // to the end of a list on an older bucket (which is OK), or to the end of a list
+ // on a newer bucket. (which will require rewalking the entire list as we may have
+ // missed elements of the list that need to be found. On the newer buckets we may
+ // find an end sentinel for the new bucket list, or an end sentinel for a different
+ // bucket from the current buckets.)
+
+ static bool IsEndSentinel(PTR_VolatileEntry entry)
+ {
+ return IsEndSentinel(dac_cast<TADDR>(entry));
+ }
+
+ static bool IsEndSentinel(TADDR value)
+ {
+ return !!(value & 1);
+ }
+
+ static TADDR InitialEndSentinel()
+ {
+ return 1;
+ }
+
+ static TADDR IncrementBaseEndSentinel(TADDR previousSentinel)
+ {
+ auto result = previousSentinel + 2;
+ _ASSERTE(IsEndSentinel(result));
+ return result;
+ }
+
+ static TADDR ComputeEndSentinel(TADDR baseEndSentinel, DWORD bucketIndex)
+ {
+ return ((TADDR)bucketIndex << 6) | baseEndSentinel;
+ }
+
+ static DWORD BucketIndexFromEndSentinel(TADDR endSentinel)
+ {
+ _ASSERTE(IsEndSentinel(endSentinel));
+ return (DWORD)endSentinel >> 6;
+ }
+
+ static DWORD BucketsAgeFromEndSentinel(TADDR endSentinel)
+ {
+ _ASSERTE(IsEndSentinel(endSentinel));
+ return ((DWORD)endSentinel & 0x3E) >> 1;
+ }
+
+ static DWORD MaxBucketCountRepresentableWithEndSentinel()
+ {
+#ifdef TARGET_64BIT
+ return 0xFFFFFFFF;
+#else
+ return 0x03FFFFFF; // Bucket age and the IsEndSentinel bit take up 6 bits
+#endif
+ }
+
+ static DWORD MaxBucketAge()
+ {
+ return 0x3E >> 1;
+ }
+
+ static bool AcceptableEndSentinel(PTR_VolatileEntry entry, TADDR expectedEndSentinel)
+ {
+ _ASSERTE(expectedEndSentinel != NULL);
+ _ASSERTE(entry != NULL);
+
+ TADDR endSentinelEntry = dac_cast<TADDR>(entry);
+
+ // Exactly matching the end sentinel
+ if (endSentinelEntry == expectedEndSentinel)
+ return true;
+
+ // An end sentinel from an earlier BucketAge is also OK. This can happen when the bucket in the
+ // new set of buckets temporarily has the remnants of a list from the old buckets.
+ if (BucketsAgeFromEndSentinel(endSentinelEntry) < BucketsAgeFromEndSentinel(expectedEndSentinel))
+ return true;
+
+ // If we reach this point, we either have found an end sentinel from a higher age set of buckets
+ // OR we've found the end from the wrong list on the current bucket.
+ _ASSERTE((BucketsAgeFromEndSentinel(endSentinelEntry) > BucketsAgeFromEndSentinel(expectedEndSentinel)) ||
+ (BucketsAgeFromEndSentinel(endSentinelEntry) == BucketsAgeFromEndSentinel(expectedEndSentinel)));
+
+ return false;
+ }
+
protected:
// This opaque structure provides enumeration context when walking the set of entries which share a common
// hash code. Initialized by BaseFindFirstEntryByHash and read/updated by BaseFindNextEntryByHash.
TADDR m_pEntry; // The entry the caller is currently looking at (or NULL to begin
// with). This is a VolatileEntry* and should always be a target address
// not a DAC PTR_.
+ TADDR m_expectedEndSentinel; // A marker indicating which bucket list is being walked
+ // The algorihm may walk off of one list to another when the list is
+ // being updated, and this allows graceful handling of that case.
DPTR(PTR_VolatileEntry) m_curBuckets; // The bucket table we are working with.
};
{
SUPPORTS_DAC;
- return m_pBuckets;
+ return VolatileLoadWithoutBarrier(&m_pBuckets);
}
// our bucket table uses two extra slots - slot [0] contains the length of the table,
// slot [1] will contain the next version of the table if it resizes
static const int SLOT_LENGTH = 0;
static const int SLOT_NEXT = 1;
- // normal slots start at slot #2
- static const int SKIP_SPECIAL_SLOTS = 2;
+ static const int SLOT_ENDSENTINEL = 2;
+ // normal slots start at slot #3
+ static const int SKIP_SPECIAL_SLOTS = 3;
static DWORD GetLength(DPTR(PTR_VolatileEntry) buckets)
{
return (DWORD)dac_cast<TADDR>(buckets[SLOT_LENGTH]);
}
+ static TADDR BaseEndSentinel(DPTR(PTR_VolatileEntry) buckets)
+ {
+ return dac_cast<TADDR>(buckets[SLOT_ENDSENTINEL]);
+ }
+
static DPTR(PTR_VolatileEntry) GetNext(DPTR(PTR_VolatileEntry) buckets)
{
- return dac_cast<DPTR(PTR_VolatileEntry)>(buckets[SLOT_NEXT]);
+ return dac_cast<DPTR(PTR_VolatileEntry)>(VolatileLoadWithoutBarrier(&buckets[SLOT_NEXT]));
}
// Loader heap provided at construction time. May be NULL (in which case m_pModule must *not* be NULL).
m_cEntries = 0;
PTR_VolatileEntry* pBuckets = (PTR_VolatileEntry*)(void*)GetHeap()->AllocMem(cbBuckets);
((size_t*)pBuckets)[SLOT_LENGTH] = cInitialBuckets;
+ ((size_t*)pBuckets)[SLOT_ENDSENTINEL] = InitialEndSentinel();
+
+ TADDR endSentinel = BaseEndSentinel(pBuckets);
+
+ // All buckets are initially empty.
+ // Note: Memory allocated on loader heap is zero filled, and since we need end sentinels, fill them in now
+ for (DWORD i = 0; i < cInitialBuckets; i++)
+ {
+ DWORD dwCurBucket = i + SKIP_SPECIAL_SLOTS;
+ pBuckets[dwCurBucket] = dac_cast<PTR_VolatileEntry>(ComputeEndSentinel(endSentinel, dwCurBucket));
+ }
// publish after setting the length
VolatileStore(&m_pBuckets, pBuckets);
// Make the new bucket table larger by the scale factor requested by the subclass (but also prime).
DWORD cNewBuckets = NextLargestPrime(cBuckets * SCALE_FACTOR);
+
+ // If NextLargestPrime is no longer incrementing,
+ // or the cBuckets can't be represented as a DWORD
+ // or the new bucket count can't be represented within and EndSentinel,
+ // or the bucket age has already reached its maximum value,
+ // just don't expand the table
+ if ((cNewBuckets == cBuckets) ||
+ (cBuckets > UINT32_MAX - SKIP_SPECIAL_SLOTS) ||
+ (SKIP_SPECIAL_SLOTS + cBuckets > MaxBucketCountRepresentableWithEndSentinel()) ||
+ (BucketsAgeFromEndSentinel(BaseEndSentinel(curBuckets)) == MaxBucketAge()))
+ return;
+
// two extra slots - slot [0] contains the length of the table,
// slot [1] will contain the next version of the table if it resizes
S_SIZE_T cbNewBuckets = (S_SIZE_T(cNewBuckets) + S_SIZE_T(SKIP_SPECIAL_SLOTS)) * S_SIZE_T(sizeof(PTR_VolatileEntry));
// element 0 stores the length of the table
((size_t*)pNewBuckets)[SLOT_LENGTH] = cNewBuckets;
+ ((size_t*)pNewBuckets)[SLOT_ENDSENTINEL] = IncrementBaseEndSentinel(BaseEndSentinel(curBuckets));
// element 1 stores the next version of the table (after length is written)
- // NOTE: DAC does not call add/grow, so this cast is ok.
- VolatileStore(&((PTR_VolatileEntry**)curBuckets)[SLOT_NEXT], pNewBuckets);
+
+ TADDR newEndSentinel = BaseEndSentinel(pNewBuckets);
+
+ _ASSERTE(BaseEndSentinel(curBuckets) != BaseEndSentinel(pNewBuckets));
+
+ // It is acceptable to walk a chain and find a sentinel from an older bucket, but not vice versa
+ _ASSERTE(AcceptableEndSentinel(dac_cast<PTR_VolatileEntry>(BaseEndSentinel(curBuckets)), BaseEndSentinel(pNewBuckets)));
+ _ASSERTE(!AcceptableEndSentinel(dac_cast<PTR_VolatileEntry>(BaseEndSentinel(pNewBuckets)), BaseEndSentinel(curBuckets)));
// All buckets are initially empty.
- // Note: Memory allocated on loader heap is zero filled
- // memset(pNewBuckets, 0, cNewBuckets * sizeof(PTR_VolatileEntry));
+ // Note: Memory allocated on loader heap is zero filled, and since we need end sentinels, fill them in now
+ for (DWORD i = 0; i < cNewBuckets; i++)
+ {
+ DWORD dwCurBucket = i + SKIP_SPECIAL_SLOTS;
+ pNewBuckets[dwCurBucket] = dac_cast<PTR_VolatileEntry>(ComputeEndSentinel(newEndSentinel, dwCurBucket));
+ }
+
+ // NOTE: DAC does not call add/grow, so this cast is ok.
+ VolatileStore(&((PTR_VolatileEntry**)curBuckets)[SLOT_NEXT], pNewBuckets);
// Run through the old table and transfer all the entries. Be sure not to mess with the integrity of the
// old table while we are doing this, as there can be concurrent readers!
DWORD dwCurBucket = i + SKIP_SPECIAL_SLOTS;
PTR_VolatileEntry pEntry = curBuckets[dwCurBucket];
- while (pEntry != NULL)
+ while (!IsEndSentinel(pEntry))
{
DWORD dwNewBucket = (pEntry->m_iHashValue % cNewBuckets) + SKIP_SPECIAL_SLOTS;
PTR_VolatileEntry pNextEntry = pEntry->m_pNextEntry;
PTR_VolatileEntry pTail = pNewBuckets[dwNewBucket];
// make the pEntry reachable in the new bucket, together with all the chain (that is temporary and ok)
- if (pTail == NULL)
+ if (IsEndSentinel(pTail))
{
pNewBuckets[dwNewBucket] = pEntry;
}
else
{
- while (pTail->m_pNextEntry)
+ while (!IsEndSentinel(pTail->m_pNextEntry))
{
pTail = pTail->m_pNextEntry;
}
VolatileStore(&curBuckets[dwCurBucket], pNextEntry);
// drop the rest of the bucket after old table starts referring to it
- VolatileStore(&pEntry->m_pNextEntry, (PTR_VolatileEntry)NULL);
+ // NOTE: this can cause a race condition where a reader thread (which is unlocked relative to this logic)
+ // can be walking this list, and stop early, failing to walk the rest of the chain. We use an incrementing
+ // end sentinel to detect that case, and repeat the entire walk.
+ VolatileStore(&pEntry->m_pNextEntry, dac_cast<PTR_VolatileEntry>(ComputeEndSentinel(newEndSentinel, dwNewBucket)));
pEntry = pNextEntry;
}
// Point at the first entry in the bucket chain that stores entries with the given hash code.
PTR_VolatileEntry pEntry = VolatileLoadWithoutBarrier(&curBuckets[dwBucket]);
+ TADDR expectedEndSentinel = ComputeEndSentinel(BaseEndSentinel(curBuckets), dwBucket);
// Walk the bucket chain one entry at a time.
- while (pEntry)
+ while (!IsEndSentinel(pEntry))
{
if (pEntry->m_iHashValue == iHash)
{
// BaseFindNextEntryByHash can pick up the search where it left off.
pContext->m_pEntry = dac_cast<TADDR>(pEntry);
pContext->m_curBuckets = curBuckets;
+ pContext->m_expectedEndSentinel = dac_cast<TADDR>(expectedEndSentinel);
// Return the address of the sub-classes' embedded entry structure.
return VALUE_FROM_VOLATILE_ENTRY(pEntry);
pEntry = VolatileLoadWithoutBarrier(&pEntry->m_pNextEntry);
}
+ if (!AcceptableEndSentinel(pEntry, expectedEndSentinel))
+ {
+ // If we hit this logic, we've managed to hit a case where the linked list was in the process of being
+ // moved to a new set of buckets while we were walking the list, and we walked part of the list of the
+ // bucket in the old hash table (which is fine), and part of the list in the new table, which may not
+ // be the correct bucket to walk. Most notably, the situation that can cause this will cause the list in
+ // the old bucket to be missing items. Restart the lookup, as the linked list is unlikely to still be under
+ // edit a second time.
+ continue;
+ }
+
// in a rare case if resize is in progress, look in the new table as well.
// if existing entry is not in the old table, it must be in the new
// since we unlink it from old only after linking into the new.
iHash = pVolatileEntry->m_iHashValue;
// Iterate over the rest ot the bucket chain.
- while ((pVolatileEntry = VolatileLoadWithoutBarrier(&pVolatileEntry->m_pNextEntry)) != nullptr)
+ while (!IsEndSentinel(pVolatileEntry = VolatileLoadWithoutBarrier(&pVolatileEntry->m_pNextEntry)))
{
if (pVolatileEntry->m_iHashValue == iHash)
{
}
}
+ if (!AcceptableEndSentinel(pVolatileEntry, pContext->m_expectedEndSentinel))
+ {
+ // If we hit this logic, we've managed to hit a case where the linked list was in the process of being
+ // moved to a new set of buckets while we were walking the list, and we walked part of the list of the
+ // bucket in the old hash table (which is fine), and part of the list in the new table, which may not
+ // be the correct bucket to walk. Most notably, the situation that can cause this will cause the list in
+ // the old bucket to be missing items. Restart the lookup, as the linked list is unlikely to still be under
+ // edit a second time.
+ return BaseFindFirstEntryByHashCore(pContext->m_curBuckets, iHash, pContext);
+ }
+
// check for next table must happen after we looked through the current.
VolatileLoadBarrier();
{
//+2 to skip "length" and "next" slots
PTR_VolatileEntry pEntry = curBuckets[i + SKIP_SPECIAL_SLOTS];
- while (pEntry.IsValid())
+ while (!IsEndSentinel(pEntry) && pEntry.IsValid())
{
pEntry.EnumMem();
}
// If we found an entry in the last step return with it.
- if (m_pEntry)
+ if (!IsEndSentinel(m_pEntry))
return VALUE_FROM_VOLATILE_ENTRY(dac_cast<PTR_VolatileEntry>(m_pEntry));
// Otherwise we found the end of a bucket chain. Increment the current bucket and, if there are
// buckets left to scan go back around again.
+ m_pEntry = NULL;
m_dwBucket++;
}
projects / platform / upstream / dotnet / runtime.git / commitdiff