}
}
-inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
+inline
+static BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
{
return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
}
// loh
{3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
// poh
- // TODO: tuning https://github.com/dotnet/runtime/issues/13739
{3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
},
// loh
{3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
// poh
- // TODO: tuning https://github.com/dotnet/runtime/issues/13739
{3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
},
};
heap_segment_allocated (pseg) = heap_segment_mem (pseg) + Align (min_obj_size, get_alignment_constant (FALSE));
heap_segment_used (pseg) = heap_segment_allocated (pseg) - plug_skew;
- generation_of (max_generation)->free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
- generation_of (loh_generation)->free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
- generation_of (poh_generation)->free_list_allocator = allocator(NUM_POH_ALIST, BASE_POH_ALIST, poh_alloc_list);
+ generation_of (max_generation)->free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST_BITS, gen2_alloc_list);
+ generation_of (loh_generation)->free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST_BITS, loh_alloc_list);
+ generation_of (poh_generation)->free_list_allocator = allocator(NUM_POH_ALIST, BASE_POH_ALIST_BITS, poh_alloc_list);
for (int gen_num = 0; gen_num < total_generation_count; gen_num++)
{
}
#endif //VERIFY_HEAP && BACKGROUND_GC
-allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
+allocator::allocator (unsigned int num_b, int fbb, alloc_list* b)
{
assert (num_b < MAX_BUCKET_COUNT);
num_buckets = num_b;
- frst_bucket_size = fbs;
+ first_bucket_bits = fbb;
buckets = b;
}
}
//always thread to the end.
-void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
+void allocator::thread_item (uint8_t* item, size_t size)
{
+ unsigned int a_l_number = first_suitable_bucket(size);
+ alloc_list* al = &alloc_list_of (a_l_number);
+ uint8_t*& head = al->alloc_list_head();
+ uint8_t*& tail = al->alloc_list_tail();
+
free_list_slot (item) = 0;
free_list_undo (item) = UNDO_EMPTY;
assert (item != head);
if (head == 0)
{
- head = item;
- }
- //TODO: This shouldn't happen anymore - verify that's the case.
- //the following is necessary because the last free element
- //may have been truncated, and tail isn't updated.
- else if (free_list_slot (head) == 0)
- {
- free_list_slot (head) = item;
+ head = item;
}
else
{
+ assert ((free_list_slot(head) != 0) || (tail == head));
assert (item != tail);
assert (free_list_slot(tail) == 0);
+
free_list_slot (tail) = item;
}
- tail = item;
-}
-void allocator::thread_item (uint8_t* item, size_t size)
-{
- size_t sz = frst_bucket_size;
- unsigned int a_l_number = 0;
-
- for (; a_l_number < (num_buckets-1); a_l_number++)
- {
- if (size < sz)
- {
- break;
- }
- sz = sz * 2;
- }
- alloc_list* al = &alloc_list_of (a_l_number);
- thread_free_item (item,
- al->alloc_list_head(),
- al->alloc_list_tail());
+ tail = item;
}
void allocator::thread_item_front (uint8_t* item, size_t size)
{
- //find right free list
- size_t sz = frst_bucket_size;
- unsigned int a_l_number = 0;
- for (; a_l_number < (num_buckets-1); a_l_number++)
- {
- if (size < sz)
- {
- break;
- }
- sz = sz * 2;
- }
+ unsigned int a_l_number = first_suitable_bucket (size);
alloc_list* al = &alloc_list_of (a_l_number);
+
free_list_slot (item) = al->alloc_list_head();
free_list_undo (item) = UNDO_EMPTY;
BOOL can_fit = FALSE;
generation* gen = generation_of (gen_number);
allocator* gen_allocator = generation_allocator (gen);
- size_t sz_list = gen_allocator->first_bucket_size();
- for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
+
+ for (unsigned int a_l_idx = gen_allocator->first_suitable_bucket(size); a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
{
- if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
- {
- uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
- uint8_t* prev_free_item = 0;
+ uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
+ uint8_t* prev_free_item = 0;
- while (free_list != 0)
+ while (free_list != 0)
+ {
+ dprintf (3, ("considering free list %Ix", (size_t)free_list));
+ size_t free_list_size = unused_array_size (free_list);
+ if ((size + Align (min_obj_size, align_const)) <= free_list_size)
{
- dprintf (3, ("considering free list %Ix", (size_t)free_list));
- size_t free_list_size = unused_array_size (free_list);
- if ((size + Align (min_obj_size, align_const)) <= free_list_size)
- {
- dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
- (size_t)free_list, free_list_size));
-
- gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
- // We ask for more Align (min_obj_size)
- // to make sure that we can insert a free object
- // in adjust_limit will set the limit lower
- size_t limit = limit_from_size (size, flags, free_list_size, gen_number, align_const);
-
- uint8_t* remain = (free_list + limit);
- size_t remain_size = (free_list_size - limit);
- if (remain_size >= Align(min_free_list, align_const))
- {
- make_unused_array (remain, remain_size);
- gen_allocator->thread_item_front (remain, remain_size);
- assert (remain_size >= Align (min_obj_size, align_const));
- }
- else
- {
- //absorb the entire free list
- limit += remain_size;
- }
- generation_free_list_space (gen) -= limit;
+ dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
+ (size_t)free_list, free_list_size));
- adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
+ gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
+ // We ask for more Align (min_obj_size)
+ // to make sure that we can insert a free object
+ // in adjust_limit will set the limit lower
+ size_t limit = limit_from_size (size, flags, free_list_size, gen_number, align_const);
- can_fit = TRUE;
- goto end;
- }
- else if (gen_allocator->discard_if_no_fit_p())
+ uint8_t* remain = (free_list + limit);
+ size_t remain_size = (free_list_size - limit);
+ if (remain_size >= Align(min_free_list, align_const))
{
- assert (prev_free_item == 0);
- dprintf (3, ("couldn't use this free area, discarding"));
- generation_free_obj_space (gen) += free_list_size;
-
- gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
- generation_free_list_space (gen) -= free_list_size;
+ make_unused_array (remain, remain_size);
+ gen_allocator->thread_item_front (remain, remain_size);
+ assert (remain_size >= Align (min_obj_size, align_const));
}
else
{
- prev_free_item = free_list;
+ //absorb the entire free list
+ limit += remain_size;
}
- free_list = free_list_slot (free_list);
+ generation_free_list_space (gen) -= limit;
+
+ adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
+
+ can_fit = TRUE;
+ goto end;
+ }
+ else if (gen_allocator->discard_if_no_fit_p())
+ {
+ assert (prev_free_item == 0);
+ dprintf (3, ("couldn't use this free area, discarding"));
+ generation_free_obj_space (gen) += free_list_size;
+
+ gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
+ generation_free_list_space (gen) -= free_list_size;
+ }
+ else
+ {
+ prev_free_item = free_list;
}
+ free_list = free_list_slot (free_list);
}
- sz_list = sz_list * 2;
}
end:
return can_fit;
#ifdef BACKGROUND_GC
int cookie = -1;
#endif //BACKGROUND_GC
- size_t sz_list = allocator->first_bucket_size();
- for (unsigned int a_l_idx = 0; a_l_idx < allocator->number_of_buckets(); a_l_idx++)
+
+ for (unsigned int a_l_idx = allocator->first_suitable_bucket(size); a_l_idx < allocator->number_of_buckets(); a_l_idx++)
{
- if ((size < sz_list) || (a_l_idx == (allocator->number_of_buckets()-1)))
+ uint8_t* free_list = allocator->alloc_list_head_of (a_l_idx);
+ uint8_t* prev_free_item = 0;
+ while (free_list != 0)
{
- uint8_t* free_list = allocator->alloc_list_head_of (a_l_idx);
- uint8_t* prev_free_item = 0;
- while (free_list != 0)
- {
- dprintf (3, ("considering free list %Ix", (size_t)free_list));
+ dprintf (3, ("considering free list %Ix", (size_t)free_list));
- size_t free_list_size = unused_array_size(free_list);
+ size_t free_list_size = unused_array_size(free_list);
- ptrdiff_t diff = free_list_size - size;
+ ptrdiff_t diff = free_list_size - size;
#ifdef FEATURE_LOH_COMPACTION
- diff -= loh_pad;
+ diff -= loh_pad;
#endif //FEATURE_LOH_COMPACTION
- // must fit exactly or leave formattable space
- if ((diff == 0) || (diff > (ptrdiff_t)Align (min_obj_size, align_const)))
- {
+ // must fit exactly or leave formattable space
+ if ((diff == 0) || (diff > (ptrdiff_t)Align (min_obj_size, align_const)))
+ {
#ifdef BACKGROUND_GC
- cookie = bgc_alloc_lock->uoh_alloc_set (free_list);
- bgc_track_uoh_alloc();
+ cookie = bgc_alloc_lock->uoh_alloc_set (free_list);
+ bgc_track_uoh_alloc();
#endif //BACKGROUND_GC
- //unlink the free_item
- allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
+ //unlink the free_item
+ allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
- // Substract min obj size because limit_from_size adds it. Not needed for LOH
- size_t limit = limit_from_size (size - Align(min_obj_size, align_const), flags, free_list_size,
- gen_number, align_const);
+ // Substract min obj size because limit_from_size adds it. Not needed for LOH
+ size_t limit = limit_from_size (size - Align(min_obj_size, align_const), flags, free_list_size,
+ gen_number, align_const);
#ifdef FEATURE_LOH_COMPACTION
- if (loh_pad)
- {
- make_unused_array (free_list, loh_pad);
- limit -= loh_pad;
- free_list += loh_pad;
- free_list_size -= loh_pad;
- }
+ if (loh_pad)
+ {
+ make_unused_array (free_list, loh_pad);
+ limit -= loh_pad;
+ free_list += loh_pad;
+ free_list_size -= loh_pad;
+ }
#endif //FEATURE_LOH_COMPACTION
- uint8_t* remain = (free_list + limit);
- size_t remain_size = (free_list_size - limit);
- if (remain_size != 0)
- {
- assert (remain_size >= Align (min_obj_size, align_const));
- make_unused_array (remain, remain_size);
- }
- if (remain_size >= Align(min_free_list, align_const))
- {
- loh_thread_gap_front (remain, remain_size, gen);
- assert (remain_size >= Align (min_obj_size, align_const));
- }
- else
- {
- generation_free_obj_space (gen) += remain_size;
- }
- generation_free_list_space (gen) -= free_list_size;
- dprintf (3, ("found fit on loh at %Ix", free_list));
+ uint8_t* remain = (free_list + limit);
+ size_t remain_size = (free_list_size - limit);
+ if (remain_size != 0)
+ {
+ assert (remain_size >= Align (min_obj_size, align_const));
+ make_unused_array (remain, remain_size);
+ }
+ if (remain_size >= Align(min_free_list, align_const))
+ {
+ loh_thread_gap_front (remain, remain_size, gen);
+ assert (remain_size >= Align (min_obj_size, align_const));
+ }
+ else
+ {
+ generation_free_obj_space (gen) += remain_size;
+ }
+ generation_free_list_space (gen) -= free_list_size;
+ dprintf (3, ("found fit on loh at %Ix", free_list));
#ifdef BACKGROUND_GC
- if (cookie != -1)
- {
- bgc_uoh_alloc_clr (free_list, limit, acontext, flags, align_const, cookie, FALSE, 0);
- }
- else
+ if (cookie != -1)
+ {
+ bgc_uoh_alloc_clr (free_list, limit, acontext, flags, align_const, cookie, FALSE, 0);
+ }
+ else
#endif //BACKGROUND_GC
- {
- adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
- }
-
- //fix the limit to compensate for adjust_limit_clr making it too short
- acontext->alloc_limit += Align (min_obj_size, align_const);
- can_fit = TRUE;
- goto exit;
+ {
+ adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
}
- prev_free_item = free_list;
- free_list = free_list_slot (free_list);
+
+ //fix the limit to compensate for adjust_limit_clr making it too short
+ acontext->alloc_limit += Align (min_obj_size, align_const);
+ can_fit = TRUE;
+ goto exit;
}
+ prev_free_item = free_list;
+ free_list = free_list_slot (free_list);
}
- sz_list = sz_list * 2;
}
exit:
return can_fit;
if (((bgc_begin_size / end_size) >= 2) || (bgc_size_increased >= bgc_begin_size))
{
- if ((bgc_begin_size / end_size) > 2)
+ if ((bgc_begin_size / end_size) >= 2)
{
dprintf (3, ("alloc-ed too much before bgc started"));
}
#ifdef BACKGROUND_GC
if (can_allocate && recursive_gc_sync::background_running_p())
{
- bgc_loh_size_increased += size;
+ if (gen_number == poh_generation)
+ {
+ bgc_poh_size_increased += size;
+ }
+ else
+ {
+ bgc_loh_size_increased += size;
+ }
}
#endif //BACKGROUND_GC
}
bgc_loh_allocate_spin() :
bgc_poh_allocate_spin();
- if (spin_for_allocation >= 0)
+ if (spin_for_allocation > 0)
{
add_saved_spinlock_info (true, me_release, mt_alloc_large);
leave_spin_lock (&more_space_lock_uoh);
add_saved_spinlock_info (true, me_acquire, mt_alloc_large);
dprintf (SPINLOCK_LOG, ("[%d]spin Emsl uoh", heap_number));
}
- else
+ else if (spin_for_allocation < 0)
{
wait_for_background (awr_uoh_alloc_during_bgc, true);
}
if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
{
- size_t sz_list = gen_allocator->first_bucket_size();
- for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
+ for (unsigned int a_l_idx = gen_allocator->first_suitable_bucket(real_size * 2); a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
{
- if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
+ uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
+ uint8_t* prev_free_item = 0;
+ while (free_list != 0)
{
- uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
- uint8_t* prev_free_item = 0;
- while (free_list != 0)
- {
- dprintf (3, ("considering free list %Ix", (size_t)free_list));
+ dprintf (3, ("considering free list %Ix", (size_t)free_list));
- size_t free_list_size = unused_array_size (free_list);
+ size_t free_list_size = unused_array_size (free_list);
- if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
- old_loc, USE_PADDING_TAIL | pad_in_front))
- {
- dprintf (4, ("F:%Ix-%Id",
- (size_t)free_list, free_list_size));
+ if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
+ old_loc, USE_PADDING_TAIL | pad_in_front))
+ {
+ dprintf (4, ("F:%Ix-%Id",
+ (size_t)free_list, free_list_size));
- gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
- generation_free_list_space (gen) -= free_list_size;
- remove_gen_free (gen->gen_num, free_list_size);
+ gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
+ generation_free_list_space (gen) -= free_list_size;
+ remove_gen_free (gen->gen_num, free_list_size);
- adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
- generation_allocate_end_seg_p (gen) = FALSE;
- goto finished;
- }
- // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
- else if (discard_p || (a_l_idx == 0))
- {
- dprintf (3, ("couldn't use this free area, discarding"));
- generation_free_obj_space (gen) += free_list_size;
+ adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
+ generation_allocate_end_seg_p (gen) = FALSE;
+ goto finished;
+ }
+ // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
+ else if (discard_p || (a_l_idx == 0))
+ {
+ dprintf (3, ("couldn't use this free area, discarding"));
+ generation_free_obj_space (gen) += free_list_size;
- gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
- generation_free_list_space (gen) -= free_list_size;
- remove_gen_free (gen->gen_num, free_list_size);
- }
- else
- {
- prev_free_item = free_list;
- }
- free_list = free_list_slot (free_list);
+ gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
+ generation_free_list_space (gen) -= free_list_size;
+ remove_gen_free (gen->gen_num, free_list_size);
+ }
+ else
+ {
+ prev_free_item = free_list;
}
+ free_list = free_list_slot (free_list);
}
- sz_list = sz_list * 2;
}
//go back to the beginning of the segment list
heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
BOOL gc_heap::find_loh_free_for_no_gc()
{
allocator* loh_allocator = generation_allocator (generation_of (loh_generation));
- size_t sz_list = loh_allocator->first_bucket_size();
size_t size = loh_allocation_no_gc;
- for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
+ for (unsigned int a_l_idx = loh_allocator->first_suitable_bucket(size); a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
{
- if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
+ uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
+ while (free_list)
{
- uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
- while (free_list)
- {
- size_t free_list_size = unused_array_size(free_list);
+ size_t free_list_size = unused_array_size(free_list);
- if (free_list_size > loh_allocation_no_gc)
- {
- dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
- return TRUE;
- }
-
- free_list = free_list_slot (free_list);
+ if (free_list_size > size)
+ {
+ dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
+ return TRUE;
}
+
+ free_list = free_list_slot (free_list);
}
- sz_list = sz_list * 2;
}
return FALSE;
(1 + condemned_gen_number));
generation* older_gen = 0;
generation* consing_gen = condemned_gen1;
- alloc_list r_free_list [MAX_BUCKET_COUNT];
+ alloc_list r_free_list [MAX_SOH_BUCKET_COUNT];
size_t r_free_list_space = 0;
size_t r_free_obj_space = 0;
size_t largest_free_space = free_space;
dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
//find the first free list in range of the current segment
- size_t sz_list = gen_allocator->first_bucket_size();
- unsigned int a_l_idx = 0;
uint8_t* free_list = 0;
+ unsigned int a_l_idx = gen_allocator->first_suitable_bucket(eph_gen_starts);
for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
{
- if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
+ free_list = gen_allocator->alloc_list_head_of (a_l_idx);
+ while (free_list)
{
- free_list = gen_allocator->alloc_list_head_of (a_l_idx);
- while (free_list)
+ if ((free_list >= first_address) &&
+ (free_list < end_address) &&
+ (unused_array_size (free_list) >= eph_gen_starts))
{
- if ((free_list >= first_address) &&
- (free_list < end_address) &&
- (unused_array_size (free_list) >= eph_gen_starts))
- {
- goto next;
- }
- else
- {
- free_list = free_list_slot (free_list);
- }
+ goto next;
+ }
+ else
+ {
+ free_list = free_list_slot (free_list);
}
}
}
#ifdef MULTIPLE_HEAPS
gc_heap::n_heaps = nhp;
- // TODO: tuning https://github.com/dotnet/runtime/issues/13739
hr = gc_heap::initialize_gc (seg_size, large_seg_size /*loh_segment_size*/, large_seg_size /*poh_segment_size*/, nhp);
#else
- // TODO: tuning https://github.com/dotnet/runtime/issues/13739
hr = gc_heap::initialize_gc (seg_size, large_seg_size /*loh_segment_size*/, large_seg_size /*poh_segment_size*/);
#endif //MULTIPLE_HEAPS
projects / platform / upstream / dotnet / runtime.git / commitdiff