1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
8 // GC automatically manages memory allocated by managed code.
9 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
11 // This file includes both the code for GC and the allocator. The most common
12 // case for a GC to be triggered is from the allocator code. See
13 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
15 // Entry points for the allocator are GCHeap::Alloc* which are called by the
16 // allocation helpers in gcscan.cpp
21 #if defined(TARGET_AMD64) && defined(TARGET_WINDOWS)
27 // We just needed a simple random number generator for testing.
33 static uint64_t get_rand()
35 x = (314159269*x+278281) & 0x7FFFFFFF;
39 // obtain random number in the range 0 .. r-1
40 static uint64_t get_rand(uint64_t r) {
42 uint64_t x = (uint64_t)((get_rand() * r) >> 31);
47 uint64_t gc_rand::x = 0;
49 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
50 BOOL bgc_heap_walk_for_etw_p = FALSE;
51 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
53 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
54 #define commit_min_th (16*OS_PAGE_SIZE)
56 static size_t smoothed_desired_per_heap = 0;
59 #define partial_size_th 100
60 #define num_partial_refs 64
62 #define partial_size_th 100
63 #define num_partial_refs 32
66 #define demotion_plug_len_th (6*1024*1024)
69 #define MARK_STACK_INITIAL_LENGTH 1024
71 #define MARK_STACK_INITIAL_LENGTH 128
74 #define LOH_PIN_QUEUE_LENGTH 100
75 #define LOH_PIN_DECAY 10
77 uint32_t yp_spin_count_unit = 0;
78 size_t loh_size_threshold = LARGE_OBJECT_SIZE;
80 #ifdef GC_CONFIG_DRIVEN
81 int compact_ratio = 0;
82 #endif //GC_CONFIG_DRIVEN
84 // See comments in reset_memory.
85 BOOL reset_mm_p = TRUE;
88 bool g_built_with_svr_gc = true;
90 bool g_built_with_svr_gc = false;
91 #endif // FEATURE_SVR_GC
93 #if defined(BUILDENV_DEBUG)
94 uint8_t g_build_variant = 0;
95 #elif defined(BUILDENV_CHECKED)
96 uint8_t g_build_variant = 1;
98 uint8_t g_build_variant = 2;
99 #endif //BUILDENV_DEBUG
101 VOLATILE(int32_t) g_no_gc_lock = -1;
103 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
104 const char * const allocation_state_str[] = {
113 "try_free_full_seg_in_bgc",
114 "try_free_after_bgc",
117 "acquire_seg_after_cg",
118 "acquire_seg_after_bgc",
119 "check_and_wait_for_bgc",
120 "trigger_full_compact_gc",
121 "trigger_ephemeral_gc",
122 "trigger_2nd_ephemeral_gc",
126 const char * const msl_take_state_str[] = {
142 #endif //TRACE_GC && !DACCESS_COMPILE
145 // Keep this in sync with the definition of gc_reason
146 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
147 static const char* const str_gc_reasons[] =
159 "induced_compacting",
162 "lowmemory_host_blocking"
165 static const char* const str_gc_pause_modes[] =
170 "sustained_low_latency",
173 #endif //DT_LOG || TRACE_GC
176 BOOL is_induced (gc_reason reason)
178 return ((reason == reason_induced) ||
179 (reason == reason_induced_noforce) ||
180 (reason == reason_lowmemory) ||
181 (reason == reason_lowmemory_blocking) ||
182 (reason == reason_induced_compacting) ||
183 (reason == reason_lowmemory_host) ||
184 (reason == reason_lowmemory_host_blocking));
188 BOOL is_induced_blocking (gc_reason reason)
190 return ((reason == reason_induced) ||
191 (reason == reason_lowmemory_blocking) ||
192 (reason == reason_induced_compacting) ||
193 (reason == reason_lowmemory_host_blocking));
196 gc_oh_num gen_to_oh(int gen)
201 return gc_oh_num::soh;
203 return gc_oh_num::soh;
205 return gc_oh_num::soh;
207 return gc_oh_num::loh;
209 return gc_oh_num::poh;
211 return gc_oh_num::none;
215 #ifndef DACCESS_COMPILE
220 uint64_t GetHighPrecisionTimeStamp()
222 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
224 return (uint64_t)((double)ts * qpf_us);
227 uint64_t RawGetHighPrecisionTimeStamp()
229 return (uint64_t)GCToOSInterface::QueryPerformanceCounter();
233 #ifdef BGC_SERVO_TUNING
234 bool gc_heap::bgc_tuning::enable_fl_tuning = false;
235 uint32_t gc_heap::bgc_tuning::memory_load_goal = 0;
236 uint32_t gc_heap::bgc_tuning::memory_load_goal_slack = 0;
237 uint64_t gc_heap::bgc_tuning::available_memory_goal = 0;
238 bool gc_heap::bgc_tuning::panic_activated_p = false;
239 double gc_heap::bgc_tuning::accu_error_panic = 0.0;
240 double gc_heap::bgc_tuning::above_goal_kp = 0.0;
241 double gc_heap::bgc_tuning::above_goal_ki = 0.0;
242 bool gc_heap::bgc_tuning::enable_kd = false;
243 bool gc_heap::bgc_tuning::enable_ki = false;
244 bool gc_heap::bgc_tuning::enable_smooth = false;
245 bool gc_heap::bgc_tuning::enable_tbh = false;
246 bool gc_heap::bgc_tuning::enable_ff = false;
247 bool gc_heap::bgc_tuning::enable_gradual_d = false;
248 double gc_heap::bgc_tuning::above_goal_kd = 0.0;
249 double gc_heap::bgc_tuning::above_goal_ff = 0.0;
250 double gc_heap::bgc_tuning::num_gen1s_smooth_factor = 0.0;
251 double gc_heap::bgc_tuning::ml_kp = 0.0;
252 double gc_heap::bgc_tuning::ml_ki = 0.0;
253 double gc_heap::bgc_tuning::accu_error = 0.0;
255 bool gc_heap::bgc_tuning::fl_tuning_triggered = false;
257 size_t gc_heap::bgc_tuning::num_bgcs_since_tuning_trigger = 0;
259 bool gc_heap::bgc_tuning::next_bgc_p = false;
261 size_t gc_heap::bgc_tuning::gen1_index_last_bgc_end;
262 size_t gc_heap::bgc_tuning::gen1_index_last_bgc_start;
263 size_t gc_heap::bgc_tuning::gen1_index_last_bgc_sweep;
264 size_t gc_heap::bgc_tuning::actual_num_gen1s_to_trigger;
266 gc_heap::bgc_tuning::tuning_calculation gc_heap::bgc_tuning::gen_calc[2];
267 gc_heap::bgc_tuning::tuning_stats gc_heap::bgc_tuning::gen_stats[2];
268 gc_heap::bgc_tuning::bgc_size_data gc_heap::bgc_tuning::current_bgc_end_data[2];
270 size_t gc_heap::bgc_tuning::last_stepping_bgc_count = 0;
271 uint32_t gc_heap::bgc_tuning::last_stepping_mem_load = 0;
272 uint32_t gc_heap::bgc_tuning::stepping_interval = 0;
273 bool gc_heap::bgc_tuning::use_stepping_trigger_p = true;
274 double gc_heap::bgc_tuning::gen2_ratio_correction = 0.0;
275 double gc_heap::bgc_tuning::ratio_correction_step = 0.0;
277 int gc_heap::saved_bgc_tuning_reason = -1;
278 #endif //BGC_SERVO_TUNING
281 size_t round_up_power2 (size_t size)
283 // Get the 0-based index of the most-significant bit in size-1.
284 // If the call failed (because size-1 is zero), size must be 1,
285 // so return 1 (because 1 rounds up to itself).
286 DWORD highest_set_bit_index;
293 &highest_set_bit_index, size - 1)) { return 1; }
295 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
296 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
297 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
298 // number of bits shifts in zeros from the right, resulting in an output of zero.
299 return static_cast<size_t>(2) << highest_set_bit_index;
303 size_t round_down_power2 (size_t size)
305 // Get the 0-based index of the most-significant bit in size.
306 // If the call failed, size must be zero so return zero.
307 DWORD highest_set_bit_index;
314 &highest_set_bit_index, size)) { return 0; }
316 // Left-shift 1 by highest_set_bit_index to get back a value containing only
317 // the most-significant set bit of size, i.e. size rounded down
318 // to the next power-of-two value.
319 return static_cast<size_t>(1) << highest_set_bit_index;
322 // Get the 0-based index of the most-significant bit in the value.
323 // Returns -1 if the input value is zero (i.e. has no set bits).
325 int index_of_highest_set_bit (size_t value)
327 // Get the 0-based index of the most-significant bit in the value.
328 // If the call failed (because value is zero), return -1.
329 DWORD highest_set_bit_index;
336 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
340 int relative_index_power2_plug (size_t power2)
342 int index = index_of_highest_set_bit (power2);
343 assert (index <= MAX_INDEX_POWER2);
345 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
349 int relative_index_power2_free_space (size_t power2)
351 int index = index_of_highest_set_bit (power2);
352 assert (index <= MAX_INDEX_POWER2);
354 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
358 uint32_t bgc_alloc_spin_count = 140;
359 uint32_t bgc_alloc_spin_count_loh = 16;
360 uint32_t bgc_alloc_spin = 2;
363 void c_write (uint32_t& place, uint32_t value)
365 Interlocked::Exchange (&place, value);
368 #ifndef DACCESS_COMPILE
370 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
371 const size_t bgc_min_per_heap = 4*1024*1024;
373 int gc_heap::gchist_index = 0;
374 gc_mechanisms_store gc_heap::gchist[max_history_count];
376 #ifndef MULTIPLE_HEAPS
377 size_t gc_heap::total_promoted_bytes = 0;
378 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
379 int gc_heap::gchist_index_per_heap = 0;
380 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
381 #endif //MULTIPLE_HEAPS
383 void gc_heap::add_to_history_per_heap()
385 #if defined(GC_HISTORY) && defined(BACKGROUND_GC)
386 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
387 current_hist->gc_index = settings.gc_index;
388 current_hist->current_bgc_state = current_bgc_state;
389 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
390 current_hist->gc_time_ms = (uint32_t)(elapsed / 1000);
391 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
392 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
393 current_hist->gen0_start = generation_allocation_start (generation_of (0));
394 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
396 current_hist->bgc_lowest = background_saved_lowest_address;
397 current_hist->bgc_highest = background_saved_highest_address;
398 #endif //BACKGROUND_GC
399 current_hist->fgc_lowest = lowest_address;
400 current_hist->fgc_highest = highest_address;
401 current_hist->g_lowest = g_gc_lowest_address;
402 current_hist->g_highest = g_gc_highest_address;
404 gchist_index_per_heap++;
405 if (gchist_index_per_heap == max_history_count)
407 gchist_index_per_heap = 0;
409 #endif //GC_HISTORY && BACKGROUND_GC
412 void gc_heap::add_to_history()
414 #if defined(GC_HISTORY) && defined(BACKGROUND_GC)
415 gc_mechanisms_store* current_settings = &gchist[gchist_index];
416 current_settings->store (&settings);
419 if (gchist_index == max_history_count)
423 #endif //GC_HISTORY && BACKGROUND_GC
426 #endif //DACCESS_COMPILE
427 #endif //BACKGROUND_GC
429 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
430 BOOL gc_log_on = TRUE;
432 size_t gc_log_file_size = 0;
434 size_t gc_buffer_index = 0;
435 size_t max_gc_buffers = 0;
437 static CLRCriticalSection gc_log_lock;
439 // we keep this much in a buffer and only flush when the buffer is full
440 #define gc_log_buffer_size (1024*1024)
441 uint8_t* gc_log_buffer = 0;
442 size_t gc_log_buffer_offset = 0;
444 void log_va_msg(const char *fmt, va_list args)
448 const int BUFFERSIZE = 512;
449 static char rgchBuffer[BUFFERSIZE];
450 char * pBuffer = &rgchBuffer[0];
453 int buffer_start = 1;
454 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
455 buffer_start += pid_len;
456 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
457 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args);
460 msg_len = BUFFERSIZE - buffer_start;
463 msg_len += buffer_start;
465 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
468 memset (index_str, '-', 8);
469 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
470 gc_log_buffer[gc_log_buffer_offset] = '\n';
471 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
474 if (gc_buffer_index > max_gc_buffers)
476 fseek (gc_log, 0, SEEK_SET);
479 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
481 memset (gc_log_buffer, '*', gc_log_buffer_size);
482 gc_log_buffer_offset = 0;
485 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
486 gc_log_buffer_offset += msg_len;
491 void GCLog (const char *fmt, ... )
493 if (gc_log_on && (gc_log != NULL))
497 log_va_msg (fmt, args);
501 #endif // TRACE_GC && !DACCESS_COMPILE
503 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
505 BOOL gc_config_log_on = FALSE;
506 FILE* gc_config_log = NULL;
508 // we keep this much in a buffer and only flush when the buffer is full
509 #define gc_config_log_buffer_size (1*1024) // TEMP
510 uint8_t* gc_config_log_buffer = 0;
511 size_t gc_config_log_buffer_offset = 0;
513 // For config since we log so little we keep the whole history. Also it's only
514 // ever logged by one thread so no need to synchronize.
515 void log_va_msg_config(const char *fmt, va_list args)
517 const int BUFFERSIZE = 256;
518 static char rgchBuffer[BUFFERSIZE];
519 char * pBuffer = &rgchBuffer[0];
522 int buffer_start = 1;
523 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
524 assert (msg_len != -1);
525 msg_len += buffer_start;
527 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
529 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
530 fflush(gc_config_log);
531 gc_config_log_buffer_offset = 0;
534 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
535 gc_config_log_buffer_offset += msg_len;
538 void GCLogConfig (const char *fmt, ... )
540 if (gc_config_log_on && (gc_config_log != NULL))
543 va_start( args, fmt );
544 log_va_msg_config (fmt, args);
547 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
549 void GCHeap::Shutdown()
551 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
552 if (gc_log_on && (gc_log != NULL))
554 fwrite(gc_log_buffer, gc_log_buffer_offset, 1, gc_log);
557 gc_log_buffer_offset = 0;
562 #ifdef SYNCHRONIZATION_STATS
563 // Number of GCs have we done since we last logged.
564 static unsigned int gc_count_during_log;
565 // In ms. This is how often we print out stats.
566 static const unsigned int log_interval = 5000;
567 // Time (in ms) when we start a new log interval.
568 static unsigned int log_start_tick;
569 static unsigned int gc_lock_contended;
570 static int64_t log_start_hires;
571 // Cycles accumulated in SuspendEE during log_interval.
572 static uint64_t suspend_ee_during_log;
573 // Cycles accumulated in RestartEE during log_interval.
574 static uint64_t restart_ee_during_log;
575 static uint64_t gc_during_log;
576 #endif //SYNCHRONIZATION_STATS
579 init_sync_log_stats()
581 #ifdef SYNCHRONIZATION_STATS
582 if (gc_count_during_log == 0)
584 gc_heap::init_sync_stats();
585 suspend_ee_during_log = 0;
586 restart_ee_during_log = 0;
588 gc_lock_contended = 0;
590 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
591 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
593 gc_count_during_log++;
594 #endif //SYNCHRONIZATION_STATS
598 process_sync_log_stats()
600 #ifdef SYNCHRONIZATION_STATS
602 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
604 if (log_elapsed > log_interval)
606 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
607 // Print out the cycles we spent on average in each suspend and restart.
608 printf("\n_________________________________________________________________________________\n"
609 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
610 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
614 (unsigned int)(gc_during_log / gc_count_during_log),
615 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
616 (unsigned int)(restart_ee_during_log / gc_count_during_log),
617 (double)(100.0f * gc_during_log / total));
618 gc_heap::print_sync_stats(gc_count_during_log);
620 gc_count_during_log = 0;
622 #endif //SYNCHRONIZATION_STATS
625 #ifdef MULTIPLE_HEAPS
626 #ifndef DACCESS_COMPILE
627 uint32_t g_num_active_processors = 0;
631 gc_join_init_cpu_mapping = 0,
633 gc_join_generation_determined = 2,
634 gc_join_begin_mark_phase = 3,
635 gc_join_scan_dependent_handles = 4,
636 gc_join_rescan_dependent_handles = 5,
637 gc_join_scan_sizedref_done = 6,
638 gc_join_null_dead_short_weak = 7,
639 gc_join_scan_finalization = 8,
640 gc_join_null_dead_long_weak = 9,
641 gc_join_null_dead_syncblk = 10,
642 gc_join_decide_on_compaction = 11,
643 gc_join_rearrange_segs_compaction = 12,
644 gc_join_adjust_handle_age_compact = 13,
645 gc_join_adjust_handle_age_sweep = 14,
646 gc_join_begin_relocate_phase = 15,
647 gc_join_relocate_phase_done = 16,
648 gc_join_verify_objects_done = 17,
649 gc_join_start_bgc = 18,
650 gc_join_restart_ee = 19,
651 gc_join_concurrent_overflow = 20,
652 gc_join_suspend_ee = 21,
653 gc_join_bgc_after_ephemeral = 22,
654 gc_join_allow_fgc = 23,
655 gc_join_bgc_sweep = 24,
656 gc_join_suspend_ee_verify = 25,
657 gc_join_restart_ee_verify = 26,
658 gc_join_set_state_free = 27,
659 gc_r_join_update_card_bundle = 28,
660 gc_join_after_absorb = 29,
661 gc_join_verify_copy_table = 30,
662 gc_join_after_reset = 31,
663 gc_join_after_ephemeral_sweep = 32,
664 gc_join_after_profiler_heap_walk = 33,
665 gc_join_minimal_gc = 34,
666 gc_join_after_commit_soh_no_gc = 35,
667 gc_join_expand_loh_no_gc = 36,
668 gc_join_final_no_gc = 37,
669 gc_join_disable_software_write_watch = 38,
675 join_flavor_server_gc = 0,
679 #define first_thread_arrived 2
680 #pragma warning(push)
681 #pragma warning(disable:4324) // don't complain if DECLSPEC_ALIGN actually pads
682 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
684 // Shared non volatile keep on separate line to prevent eviction
687 // Keep polling/wait structures on separate line write once per join
688 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
689 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
690 Volatile<int> lock_color;
691 VOLATILE(BOOL) wait_done;
692 VOLATILE(BOOL) joined_p;
694 // Keep volatile counted locks on separate cache line write many per join
695 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
696 VOLATILE(int) join_lock;
697 VOLATILE(int) r_join_lock;
707 type_first_r_join = 3,
719 join_heap_restart = 100,
720 join_heap_r_restart = 200
725 join_structure join_struct;
728 gc_join_flavor flavor;
731 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
732 // remember join id and last thread to arrive so restart can use these
734 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
736 // counters for joins, in 1000's of clock cycles
737 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
741 BOOL init (int n_th, gc_join_flavor f)
743 dprintf (JOIN_LOG, ("Initializing join structure"));
744 join_struct.n_threads = n_th;
745 join_struct.lock_color = 0;
746 for (int i = 0; i < 3; i++)
748 if (!join_struct.joined_event[i].IsValid())
750 join_struct.joined_p = FALSE;
751 dprintf (JOIN_LOG, ("Creating join event %d", i));
752 // TODO - changing this to a non OS event
753 // because this is also used by BGC threads which are
754 // managed threads and WaitEx does not allow you to wait
755 // for an OS event on a managed thread.
756 // But we are not sure if this plays well in the hosting
758 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
759 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
763 join_struct.join_lock = join_struct.n_threads;
764 join_struct.r_join_lock = join_struct.n_threads;
765 join_struct.wait_done = FALSE;
769 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
777 dprintf (JOIN_LOG, ("Destroying join structure"));
778 for (int i = 0; i < 3; i++)
780 if (join_struct.joined_event[i].IsValid())
781 join_struct.joined_event[i].CloseEvent();
785 inline void fire_event (int heap, join_time time, join_type type, int join_id)
787 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
790 void join (gc_heap* gch, int join_id)
793 // parallel execution ends here
794 end[gch->heap_number] = get_ts();
797 assert (!join_struct.joined_p);
798 int color = join_struct.lock_color.LoadWithoutBarrier();
800 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
802 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
803 flavor, join_id, (int32_t)(join_struct.join_lock)));
805 fire_event (gch->heap_number, time_start, type_join, join_id);
807 //busy wait around the color
808 if (color == join_struct.lock_color.LoadWithoutBarrier())
811 int spin_count = 128 * yp_spin_count_unit;
812 for (int j = 0; j < spin_count; j++)
814 if (color != join_struct.lock_color.LoadWithoutBarrier())
818 YieldProcessor(); // indicate to the processor that we are spinning
821 // we've spun, and if color still hasn't changed, fall into hard wait
822 if (color == join_struct.lock_color.LoadWithoutBarrier())
824 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
825 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
827 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
829 if (dwJoinWait != WAIT_OBJECT_0)
831 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
836 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
837 if (color == join_struct.lock_color.LoadWithoutBarrier())
842 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
843 flavor, join_id, (int32_t)(join_struct.join_lock)));
846 fire_event (gch->heap_number, time_end, type_join, join_id);
849 // parallel execution starts here
850 start[gch->heap_number] = get_ts();
851 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
856 fire_event (gch->heap_number, time_start, type_last_join, join_id);
858 join_struct.joined_p = TRUE;
859 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
860 join_struct.joined_event[!color].Reset();
863 // remember the join id, the last thread arriving, the start of the sequential phase,
864 // and keep track of the cycles spent waiting in the join
865 thd = gch->heap_number;
866 start_seq = get_ts();
867 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
872 // Reverse join - first thread gets here does the work; other threads will only proceed
873 // after the work is done.
874 // Note that you cannot call this twice in a row on the same thread. Plus there's no
875 // need to call it twice in row - you should just merge the work.
876 BOOL r_join (gc_heap* gch, int join_id)
879 if (join_struct.n_threads == 1)
884 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
886 fire_event (gch->heap_number, time_start, type_join, join_id);
888 dprintf (JOIN_LOG, ("r_join() Waiting..."));
890 //busy wait around the color
892 int spin_count = 256 * yp_spin_count_unit;
893 for (int j = 0; j < spin_count; j++)
895 if (join_struct.wait_done)
899 YieldProcessor(); // indicate to the processor that we are spinning
902 // we've spun, and if color still hasn't changed, fall into hard wait
903 if (!join_struct.wait_done)
905 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
906 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
907 if (dwJoinWait != WAIT_OBJECT_0)
909 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
914 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
915 if (!join_struct.wait_done)
920 dprintf (JOIN_LOG, ("r_join() done"));
922 fire_event (gch->heap_number, time_end, type_join, join_id);
928 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
936 return GCToOSInterface::QueryPerformanceCounter();
939 void start_ts (gc_heap* gch)
941 // parallel execution ends here
942 start[gch->heap_number] = get_ts();
949 uint64_t elapsed_seq = get_ts() - start_seq;
950 uint64_t max = 0, sum = 0, wake = 0;
951 uint64_t min_ts = start[0];
952 for (int i = 1; i < join_struct.n_threads; i++)
954 if(min_ts > start[i]) min_ts = start[i];
957 for (int i = 0; i < join_struct.n_threads; i++)
959 uint64_t wake_delay = start[i] - min_ts;
960 uint64_t elapsed = end[i] - start[i];
966 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
967 uint64_t par_loss = join_struct.n_threads*max - sum;
968 double efficiency = 0.0;
970 efficiency = sum*100.0/(join_struct.n_threads*max);
972 const double ts_scale = 1e-6;
974 // enable this printf to get statistics on each individual join as it occurs
975 //printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
977 elapsed_total[id] += sum;
978 wake_total[id] += wake;
979 seq_loss_total[id] += seq_loss;
980 par_loss_total[id] += par_loss;
982 // every 10 seconds, print a summary of the time spent in each type of join
983 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
985 printf("**** summary *****\n");
986 for (int i = 0; i < 16; i++)
988 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
990 ts_scale*elapsed_total[i],
991 ts_scale*wake_total[i],
992 ts_scale*seq_loss_total[i],
993 ts_scale*par_loss_total[i],
994 ts_scale*in_join_total[i]);
995 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
997 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1001 fire_event (join_heap_restart, time_start, type_restart, -1);
1002 assert (join_struct.joined_p);
1003 join_struct.joined_p = FALSE;
1004 join_struct.join_lock = join_struct.n_threads;
1005 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1006 int color = join_struct.lock_color.LoadWithoutBarrier();
1007 join_struct.lock_color = !color;
1008 join_struct.joined_event[color].Set();
1010 fire_event (join_heap_restart, time_end, type_restart, -1);
1013 start[thd] = get_ts();
1019 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1020 return join_struct.joined_p;
1025 if (join_struct.n_threads != 1)
1027 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1028 join_struct.wait_done = TRUE;
1029 join_struct.joined_event[first_thread_arrived].Set();
1030 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1036 if (join_struct.n_threads != 1)
1038 join_struct.r_join_lock = join_struct.n_threads;
1039 join_struct.wait_done = FALSE;
1040 join_struct.joined_event[first_thread_arrived].Reset();
1047 #ifdef BACKGROUND_GC
1049 #endif //BACKGROUND_GC
1051 #endif // DACCESS_COMPILE
1053 #endif //MULTIPLE_HEAPS
1055 #define spin_and_switch(count_to_spin, expr) \
1057 for (int j = 0; j < count_to_spin; j++) \
1067 GCToOSInterface::YieldThread(0); \
1071 #if defined(BACKGROUND_GC) && !(DACCESS_COMPILE)
1073 #define max_pending_allocs 64
1075 class exclusive_sync
1077 VOLATILE(uint8_t*) rwp_object;
1078 VOLATILE(int32_t) needs_checking;
1082 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1084 // TODO - perhaps each object should be on its own cache line...
1085 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1087 int find_free_index ()
1089 for (int i = 0; i < max_pending_allocs; i++)
1091 if (alloc_objects [i] == (uint8_t*)0)
1103 spin_count = 32 * (g_num_processors - 1);
1106 for (int i = 0; i < max_pending_allocs; i++)
1108 alloc_objects [i] = (uint8_t*)0;
1114 for (int i = 0; i < max_pending_allocs; i++)
1116 if (alloc_objects [i] != (uint8_t*)0)
1118 GCToOSInterface::DebugBreak();
1123 void bgc_mark_set (uint8_t* obj)
1125 dprintf (3, ("cm: probing %Ix", obj));
1127 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1129 // If we spend too much time spending all the allocs,
1130 // consider adding a high water mark and scan up
1131 // to that; we'll need to interlock in done when
1132 // we update the high watermark.
1133 for (int i = 0; i < max_pending_allocs; i++)
1135 if (obj == alloc_objects[i])
1138 dprintf (3, ("cm: will spin"));
1139 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1146 dprintf (3, ("cm: set %Ix", obj));
1151 spin_and_switch (spin_count, (needs_checking == 0));
1156 int uoh_alloc_set (uint8_t* obj)
1158 if (!gc_heap::cm_in_progress)
1164 dprintf (3, ("uoh alloc: probing %Ix", obj));
1166 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1168 if (obj == rwp_object)
1171 spin_and_switch (spin_count, (obj != rwp_object));
1176 int cookie = find_free_index();
1180 alloc_objects[cookie] = obj;
1184 // GCToOSInterface::DebugBreak();
1187 dprintf (3, ("uoh alloc: set %Ix at %d", obj, cookie));
1193 dprintf (3, ("uoh alloc: setting %Ix will spin to acquire a free index", obj));
1194 spin_and_switch (spin_count, (find_free_index () != -1));
1201 dprintf (3, ("uoh alloc: will spin on checking %Ix", obj));
1202 spin_and_switch (spin_count, (needs_checking == 0));
1207 void bgc_mark_done ()
1209 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1213 void uoh_alloc_done_with_index (int index)
1215 dprintf (3, ("uoh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1216 assert ((index >= 0) && (index < max_pending_allocs));
1217 alloc_objects[index] = (uint8_t*)0;
1220 void uoh_alloc_done (uint8_t* obj)
1222 #ifdef BACKGROUND_GC
1223 if (!gc_heap::cm_in_progress)
1228 for (int i = 0; i < max_pending_allocs; i++)
1230 if (alloc_objects [i] == obj)
1232 uoh_alloc_done_with_index(i);
1236 #endif //BACKGROUND_GC
1240 #endif //BACKGROUND_GC && !DACCESS_COMPILE
1242 void reset_memory (uint8_t* o, size_t sizeo);
1246 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1247 static bool virtual_alloc_hardware_write_watch = false;
1248 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1250 static bool hardware_write_watch_capability = false;
1252 #ifndef DACCESS_COMPILE
1254 void hardware_write_watch_api_supported()
1256 if (GCToOSInterface::SupportsWriteWatch())
1258 hardware_write_watch_capability = true;
1259 dprintf (2, ("WriteWatch supported"));
1263 dprintf (2,("WriteWatch not supported"));
1266 #endif //!DACCESS_COMPILE
1268 inline bool can_use_hardware_write_watch()
1270 return hardware_write_watch_capability;
1273 inline bool can_use_write_watch_for_gc_heap()
1275 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1277 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1278 return can_use_hardware_write_watch();
1279 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1282 inline bool can_use_write_watch_for_card_table()
1284 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1287 return can_use_hardware_write_watch();
1292 #define mem_reserve (MEM_RESERVE)
1293 #endif //WRITE_WATCH
1295 //check if the low memory notification is supported
1297 #ifndef DACCESS_COMPILE
1299 void WaitLongerNoInstru (int i)
1301 // every 8th attempt:
1302 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1304 // if we're waiting for gc to finish, we should block immediately
1305 if (g_fSuspensionPending == 0)
1307 if (g_num_processors > 1)
1309 YieldProcessor(); // indicate to the processor that we are spinning
1311 GCToOSInterface::YieldThread (0);
1313 GCToOSInterface::Sleep (5);
1316 GCToOSInterface::Sleep (5);
1319 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1320 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1321 // It is important that the thread is going to wait for GC. Otherwise the thread
1322 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1326 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1327 // started, it is important to block until the GC thread calls set_gc_done (since it is
1328 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1329 // conditions which can otherwise occur if threads are allowed to spin in this function
1330 // (and therefore starve the GC thread) between the point when the GC thread sets the
1331 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1332 if (gc_heap::gc_started)
1334 gc_heap::wait_for_gc_done();
1337 GCToEEInterface::DisablePreemptiveGC();
1339 else if (g_fSuspensionPending > 0)
1341 g_theGCHeap->WaitUntilGCComplete();
1346 static void safe_switch_to_thread()
1348 bool cooperative_mode = gc_heap::enable_preemptive();
1350 GCToOSInterface::YieldThread(0);
1352 gc_heap::disable_preemptive(cooperative_mode);
1356 // We need the following methods to have volatile arguments, so that they can accept
1357 // raw pointers in addition to the results of the & operator on Volatile<T>.
1360 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1364 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1367 while (VolatileLoad(lock) >= 0)
1369 if ((++i & 7) && !IsGCInProgress())
1371 if (g_num_processors > 1)
1373 #ifndef MULTIPLE_HEAPS
1374 int spin_count = 32 * yp_spin_count_unit;
1375 #else //!MULTIPLE_HEAPS
1376 int spin_count = yp_spin_count_unit;
1377 #endif //!MULTIPLE_HEAPS
1378 for (int j = 0; j < spin_count; j++)
1380 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1382 YieldProcessor(); // indicate to the processor that we are spinning
1384 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1386 safe_switch_to_thread();
1391 safe_switch_to_thread();
1396 WaitLongerNoInstru(i);
1404 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1406 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1410 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1412 VolatileStore<int32_t>((int32_t*)lock, -1);
1418 static void enter_spin_lock(GCSpinLock *pSpinLock)
1420 enter_spin_lock_noinstru(&pSpinLock->lock);
1421 assert (pSpinLock->holding_thread == (Thread*)-1);
1422 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1426 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1428 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1430 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1435 static void leave_spin_lock(GCSpinLock *pSpinLock)
1437 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1438 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1439 pSpinLock->released_by_gc_p = gc_thread_p;
1440 pSpinLock->holding_thread = (Thread*) -1;
1441 if (pSpinLock->lock != -1)
1442 leave_spin_lock_noinstru(&pSpinLock->lock);
1445 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1446 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1448 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1449 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1453 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1454 //the gc thread call WaitLonger.
1455 void WaitLonger (int i
1456 #ifdef SYNCHRONIZATION_STATS
1457 , GCSpinLock* spin_lock
1458 #endif //SYNCHRONIZATION_STATS
1461 #ifdef SYNCHRONIZATION_STATS
1462 (spin_lock->num_wait_longer)++;
1463 #endif //SYNCHRONIZATION_STATS
1465 // every 8th attempt:
1466 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1469 // if we're waiting for gc to finish, we should block immediately
1470 if (!gc_heap::gc_started)
1472 #ifdef SYNCHRONIZATION_STATS
1473 (spin_lock->num_switch_thread_w)++;
1474 #endif //SYNCHRONIZATION_STATS
1475 if (g_num_processors > 1)
1477 YieldProcessor(); // indicate to the processor that we are spinning
1479 GCToOSInterface::YieldThread (0);
1481 GCToOSInterface::Sleep (5);
1484 GCToOSInterface::Sleep (5);
1487 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1488 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1489 // It is important that the thread is going to wait for GC. Otherwise the thread
1490 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1491 if (gc_heap::gc_started)
1493 gc_heap::wait_for_gc_done();
1498 #ifdef SYNCHRONIZATION_STATS
1499 (spin_lock->num_disable_preemptive_w)++;
1500 #endif //SYNCHRONIZATION_STATS
1501 GCToEEInterface::DisablePreemptiveGC();
1506 static void enter_spin_lock (GCSpinLock* spin_lock)
1510 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1513 while (spin_lock->lock >= 0)
1515 if ((++i & 7) && !gc_heap::gc_started)
1517 if (g_num_processors > 1)
1519 #ifndef MULTIPLE_HEAPS
1520 int spin_count = 32 * yp_spin_count_unit;
1521 #else //!MULTIPLE_HEAPS
1522 int spin_count = yp_spin_count_unit;
1523 #endif //!MULTIPLE_HEAPS
1524 for (int j = 0; j < spin_count; j++)
1526 if (spin_lock->lock < 0 || gc_heap::gc_started)
1528 YieldProcessor(); // indicate to the processor that we are spinning
1530 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1532 #ifdef SYNCHRONIZATION_STATS
1533 (spin_lock->num_switch_thread)++;
1534 #endif //SYNCHRONIZATION_STATS
1535 bool cooperative_mode = gc_heap::enable_preemptive ();
1537 GCToOSInterface::YieldThread(0);
1539 gc_heap::disable_preemptive (cooperative_mode);
1543 GCToOSInterface::YieldThread(0);
1548 #ifdef SYNCHRONIZATION_STATS
1550 #endif //SYNCHRONIZATION_STATS
1559 static BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1561 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1565 static void leave_spin_lock (GCSpinLock * spin_lock)
1567 spin_lock->lock = -1;
1570 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1574 bool gc_heap::enable_preemptive ()
1576 return GCToEEInterface::EnablePreemptiveGC();
1579 void gc_heap::disable_preemptive (bool restore_cooperative)
1581 if (restore_cooperative)
1583 GCToEEInterface::DisablePreemptiveGC();
1587 #endif // !DACCESS_COMPILE
1589 typedef void ** PTR_PTR;
1591 void memclr ( uint8_t* mem, size_t size)
1593 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1594 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1595 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1596 memset (mem, 0, size);
1599 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1601 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1602 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1603 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1605 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1606 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1608 // copy in groups of four pointer sized things at a time
1613 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1614 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1615 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1616 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1620 while ((size -= sz4ptr) >= sz4ptr);
1623 // still two pointer sized things or more left to copy?
1626 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1627 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1632 // still one pointer sized thing left to copy?
1635 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1640 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1642 return ((add / pitch) * pitch);
1645 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1646 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1647 // i.e, if a larger alignment matters or is beneficial, the compiler
1648 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1649 // converse - it's a heuristic for the GC to use a larger alignment.
1650 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1653 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1654 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1657 // Returns true if two pointers have the same large (double than normal) alignment.
1659 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
1661 #ifdef RESPECT_LARGE_ALIGNMENT
1662 const size_t LARGE_ALIGNMENT_MASK = 2 * DATA_ALIGNMENT - 1;
1663 return ((((size_t)p1 ^ (size_t)p2) & LARGE_ALIGNMENT_MASK) == 0);
1665 UNREFERENCED_PARAMETER(p1);
1666 UNREFERENCED_PARAMETER(p2);
1668 #endif // RESPECT_LARGE_ALIGNMENT
1671 // Determines the padding size required to fix large alignment during relocation.
1673 size_t switch_alignment_size (BOOL already_padded_p)
1675 #ifndef RESPECT_LARGE_ALIGNMENT
1676 assert (!"Should not be called");
1677 #endif // RESPECT_LARGE_ALIGNMENT
1679 if (already_padded_p)
1680 return DATA_ALIGNMENT;
1682 return Align (min_obj_size) | DATA_ALIGNMENT;
1685 #ifdef FEATURE_STRUCTALIGN
1686 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
1687 void clear_node_aligninfo (uint8_t *node);
1688 #else // FEATURE_STRUCTALIGN
1689 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
1690 void set_node_realigned (uint8_t* node);
1691 void clear_node_realigned(uint8_t* node);
1692 #endif // FEATURE_STRUCTALIGN
1695 size_t AlignQword (size_t nbytes)
1697 #ifdef FEATURE_STRUCTALIGN
1698 // This function is used to align everything on the large object
1699 // heap to an 8-byte boundary, to reduce the number of unaligned
1700 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
1701 // the compiler dictates the optimal alignment instead of having
1702 // a heuristic in the GC.
1703 return Align (nbytes);
1704 #else // FEATURE_STRUCTALIGN
1705 return (nbytes + 7) & ~7;
1706 #endif // FEATURE_STRUCTALIGN
1710 BOOL Aligned (size_t n)
1712 return (n & ALIGNCONST) == 0;
1715 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
1717 #ifdef FEATURE_STRUCTALIGN
1718 #define MAX_STRUCTALIGN OS_PAGE_SIZE
1719 #else // FEATURE_STRUCTALIGN
1720 #define MAX_STRUCTALIGN 0
1721 #endif // FEATURE_STRUCTALIGN
1723 #ifdef FEATURE_STRUCTALIGN
1725 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
1727 // The resulting alignpad must be either 0 or at least min_obj_size.
1728 // Note that by computing the following difference on unsigned types,
1729 // we can do the range check 0 < alignpad < min_obj_size with a
1730 // single conditional branch.
1731 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
1733 return requiredAlignment;
1739 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
1741 // required alignment must be a power of two
1742 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
1743 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
1744 _ASSERTE(requiredAlignment >= sizeof(void *));
1745 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
1747 // When this method is invoked for individual objects (i.e., alignmentOffset
1748 // is just the size of the PostHeader), what needs to be aligned when
1749 // we're done is the pointer to the payload of the object (which means
1750 // the actual resulting object pointer is typically not aligned).
1752 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
1753 ptrdiff_t alignpad = result - origPtr;
1755 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
1759 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
1761 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
1764 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
1766 return StructAlign (ptr, requiredAlignment) == ptr;
1770 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
1772 if (requiredAlignment == DATA_ALIGNMENT)
1774 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
1775 // alignment padding object), the worst-case alignment padding is correspondingly larger
1776 // than the required alignment.
1777 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
1781 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
1783 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
1785 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
1786 // for padding before the actual object, it also leaves space for filling a gap after the
1787 // actual object. This is needed on the large object heap, as the outer allocation functions
1788 // don't operate on an allocation context (which would have left space for the final gap).
1789 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
1792 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
1794 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
1795 if (alignedPtr != newAlloc) {
1796 make_unused_array (newAlloc, alignedPtr - newAlloc);
1798 acontext->alloc_ptr = alignedPtr + Align (size);
1802 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
1804 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
1805 if (alignedPtr != newAlloc) {
1806 make_unused_array (newAlloc, alignedPtr - newAlloc);
1808 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
1809 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
1813 #else // FEATURE_STRUCTALIGN
1814 #define ComputeMaxStructAlignPad(requiredAlignment) 0
1815 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
1816 #endif // FEATURE_STRUCTALIGN
1818 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
1820 #define CLR_SIZE ((size_t)(8*1024))
1822 #define CLR_SIZE ((size_t)(8*1024))
1825 #define END_SPACE_AFTER_GC (loh_size_threshold + MAX_STRUCTALIGN)
1827 #ifdef BACKGROUND_GC
1828 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
1830 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
1831 #endif //BACKGROUND_GC
1833 // This is always power of 2.
1834 const size_t min_segment_size_hard_limit = 1024*1024*16;
1837 size_t align_on_segment_hard_limit (size_t add)
1839 return ((size_t)(add + (min_segment_size_hard_limit - 1)) & ~(min_segment_size_hard_limit - 1));
1846 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
1847 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
1851 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
1852 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
1854 #endif // HOST_64BIT
1860 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
1861 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
1865 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
1866 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
1868 #endif // HOST_64BIT
1872 const size_t etw_allocation_tick = 100*1024;
1874 const size_t low_latency_alloc = 256*1024;
1876 const size_t fgn_check_quantum = 2*1024*1024;
1879 const int max_snoop_level = 128;
1883 //threshold of heap size to turn on card bundles.
1884 #define SH_TH_CARD_BUNDLE (40*1024*1024)
1885 #define MH_TH_CARD_BUNDLE (180*1024*1024)
1886 #endif //CARD_BUNDLE
1888 // min size to decommit to make the OS call worthwhile
1889 #define MIN_DECOMMIT_SIZE (100*OS_PAGE_SIZE)
1891 // max size to decommit per millisecond
1892 #define DECOMMIT_SIZE_PER_MILLISECOND (160*1024)
1894 // time in milliseconds between decommit steps
1895 #define DECOMMIT_TIME_STEP_MILLISECONDS (100)
1898 size_t align_on_page (size_t add)
1900 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
1904 uint8_t* align_on_page (uint8_t* add)
1906 return (uint8_t*)align_on_page ((size_t) add);
1910 size_t align_lower_page (size_t add)
1912 return (add & ~((size_t)OS_PAGE_SIZE - 1));
1916 uint8_t* align_lower_page (uint8_t* add)
1918 return (uint8_t*)align_lower_page ((size_t)add);
1922 size_t align_write_watch_lower_page (size_t add)
1924 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
1928 uint8_t* align_write_watch_lower_page (uint8_t* add)
1930 return (uint8_t*)align_lower_page ((size_t)add);
1934 BOOL power_of_two_p (size_t integer)
1936 return !(integer & (integer-1));
1940 BOOL oddp (size_t integer)
1942 return (integer & 1) != 0;
1945 // we only ever use this for WORDs.
1946 size_t logcount (size_t word)
1948 //counts the number of high bits in a 16 bit word.
1949 assert (word < 0x10000);
1951 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
1952 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
1953 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
1954 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
1958 #ifndef DACCESS_COMPILE
1960 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
1962 WriteBarrierParameters args = {};
1963 args.operation = WriteBarrierOp::StompResize;
1964 args.is_runtime_suspended = is_runtime_suspended;
1965 args.requires_upper_bounds_check = requires_upper_bounds_check;
1967 args.card_table = g_gc_card_table;
1968 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1969 args.card_bundle_table = g_gc_card_bundle_table;
1972 args.lowest_address = g_gc_lowest_address;
1973 args.highest_address = g_gc_highest_address;
1975 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1976 if (SoftwareWriteWatch::IsEnabledForGCHeap())
1978 args.write_watch_table = g_gc_sw_ww_table;
1980 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1982 GCToEEInterface::StompWriteBarrier(&args);
1985 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
1987 WriteBarrierParameters args = {};
1988 args.operation = WriteBarrierOp::StompEphemeral;
1989 args.is_runtime_suspended = true;
1990 args.ephemeral_low = ephemeral_low;
1991 args.ephemeral_high = ephemeral_high;
1992 GCToEEInterface::StompWriteBarrier(&args);
1995 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
1997 WriteBarrierParameters args = {};
1998 args.operation = WriteBarrierOp::Initialize;
1999 args.is_runtime_suspended = true;
2000 args.requires_upper_bounds_check = false;
2001 args.card_table = g_gc_card_table;
2003 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2004 args.card_bundle_table = g_gc_card_bundle_table;
2007 args.lowest_address = g_gc_lowest_address;
2008 args.highest_address = g_gc_highest_address;
2009 args.ephemeral_low = ephemeral_low;
2010 args.ephemeral_high = ephemeral_high;
2011 GCToEEInterface::StompWriteBarrier(&args);
2014 #endif // DACCESS_COMPILE
2016 //extract the low bits [0,low[ of a uint32_t
2017 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2018 //extract the high bits [high, 32] of a uint32_t
2019 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2021 // Things we need to manually initialize:
2022 // gen0 min_size - based on cache
2023 // gen0/1 max_size - based on segment size
2024 static static_data static_data_table[latency_level_last - latency_level_first + 1][total_generation_count] =
2026 // latency_level_memory_footprint
2029 {0, 0, 40000, 0.5f, 9.0f, 20.0f, (1000 * 1000), 1},
2031 {160*1024, 0, 80000, 0.5f, 2.0f, 7.0f, (10 * 1000 * 1000), 10},
2033 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, (100 * 1000 * 1000), 100},
2035 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
2037 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
2040 // latency_level_balanced
2044 #ifdef MULTIPLE_HEAPS
2048 #endif //MULTIPLE_HEAPS
2051 {256*1024, 0, 80000, 0.5f, 2.0f, 7.0f, (10 * 1000 * 1000), 10},
2053 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, (100 * 1000 * 1000), 100},
2055 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0},
2057 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2064 class CObjectHeader;
2068 class c_synchronize;
2070 #ifdef FEATURE_PREMORTEM_FINALIZATION
2071 #ifndef DACCESS_COMPILE
2073 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2074 #endif //!DACCESS_COMPILE
2075 #endif // FEATURE_PREMORTEM_FINALIZATION
2077 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2080 #ifdef USE_INTROSORT
2081 #define _sort introsort::sort
2082 #elif defined(USE_VXSORT)
2083 // in this case we have do_vxsort which takes an additional range that
2084 // all items to be sorted are contained in
2085 // so do not #define _sort
2086 #else //USE_INTROSORT
2087 #define _sort qsort1
2088 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2089 #endif //USE_INTROSORT
2091 void* virtual_alloc (size_t size);
2092 void* virtual_alloc (size_t size, bool use_large_pages_p);
2094 /* per heap static initialization */
2095 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
2096 uint32_t* gc_heap::mark_array;
2097 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
2100 uint8_t** gc_heap::g_mark_list;
2102 #ifdef PARALLEL_MARK_LIST_SORT
2103 uint8_t** gc_heap::g_mark_list_copy;
2104 #endif //PARALLEL_MARK_LIST_SORT
2106 size_t gc_heap::mark_list_size;
2107 bool gc_heap::mark_list_overflow;
2110 seg_mapping* seg_mapping_table;
2112 #ifdef FEATURE_BASICFREEZE
2113 sorted_table* gc_heap::seg_table;
2114 #endif //FEATURE_BASICFREEZE
2116 #ifdef MULTIPLE_HEAPS
2117 GCEvent gc_heap::ee_suspend_event;
2118 size_t gc_heap::min_gen0_balance_delta = 0;
2119 size_t gc_heap::min_balance_threshold = 0;
2120 #endif //MULTIPLE_HEAPS
2122 VOLATILE(BOOL) gc_heap::gc_started;
2124 #ifdef MULTIPLE_HEAPS
2126 GCEvent gc_heap::gc_start_event;
2127 bool gc_heap::gc_thread_no_affinitize_p = false;
2128 uintptr_t process_mask = 0;
2130 int gc_heap::n_heaps;
2132 gc_heap** gc_heap::g_heaps;
2134 size_t* gc_heap::g_promoted;
2137 int* gc_heap::g_mark_stack_busy;
2141 #ifdef BACKGROUND_GC
2142 size_t* gc_heap::g_bpromoted;
2143 #endif //BACKGROUND_GC
2145 BOOL gc_heap::gradual_decommit_in_progress_p = FALSE;
2146 size_t gc_heap::max_decommit_step_size = 0;
2147 #else //MULTIPLE_HEAPS
2149 size_t gc_heap::g_promoted;
2151 #ifdef BACKGROUND_GC
2152 size_t gc_heap::g_bpromoted;
2153 #endif //BACKGROUND_GC
2155 #endif //MULTIPLE_HEAPS
2157 size_t gc_heap::reserved_memory = 0;
2158 size_t gc_heap::reserved_memory_limit = 0;
2159 BOOL gc_heap::g_low_memory_status;
2161 #ifndef DACCESS_COMPILE
2162 static gc_reason gc_trigger_reason = reason_empty;
2163 #endif //DACCESS_COMPILE
2165 gc_latency_level gc_heap::latency_level = latency_level_default;
2167 gc_mechanisms gc_heap::settings;
2169 gc_history_global gc_heap::gc_data_global;
2171 uint64_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2173 CLRCriticalSection gc_heap::check_commit_cs;
2175 size_t gc_heap::current_total_committed = 0;
2177 size_t gc_heap::committed_by_oh[total_oh_count] = {0, 0, 0, 0};
2179 size_t gc_heap::current_total_committed_bookkeeping = 0;
2182 double gc_heap::short_plugs_pad_ratio = 0;
2183 #endif //SHORT_PLUGS
2185 uint64_t gc_heap::suspended_start_time = 0;
2186 uint64_t gc_heap::end_gc_time = 0;
2187 uint64_t gc_heap::total_suspended_time = 0;
2188 uint64_t gc_heap::process_start_time = 0;
2189 last_recorded_gc_info gc_heap::last_ephemeral_gc_info;
2190 last_recorded_gc_info gc_heap::last_full_blocking_gc_info;
2192 #ifdef BACKGROUND_GC
2193 last_recorded_gc_info gc_heap::last_bgc_info[2];
2194 VOLATILE(bool) gc_heap::is_last_recorded_bgc = false;
2195 VOLATILE(int) gc_heap::last_bgc_info_index = 0;
2196 #endif //BACKGROUND_GC
2198 #if defined(HOST_64BIT)
2199 #define MAX_ALLOWED_MEM_LOAD 85
2201 // consider putting this in dynamic data -
2202 // we may want different values for workstation
2204 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2206 size_t gc_heap::youngest_gen_desired_th;
2209 uint64_t gc_heap::mem_one_percent = 0;
2211 uint32_t gc_heap::high_memory_load_th = 0;
2213 uint32_t gc_heap::m_high_memory_load_th;
2215 uint32_t gc_heap::v_high_memory_load_th;
2217 bool gc_heap::is_restricted_physical_mem;
2219 uint64_t gc_heap::total_physical_mem = 0;
2221 uint64_t gc_heap::entry_available_physical_mem = 0;
2223 size_t gc_heap::heap_hard_limit = 0;
2225 size_t gc_heap::heap_hard_limit_oh[total_oh_count - 1] = {0, 0, 0};
2227 bool affinity_config_specified_p = false;
2228 #ifdef BACKGROUND_GC
2229 GCEvent gc_heap::bgc_start_event;
2231 gc_mechanisms gc_heap::saved_bgc_settings;
2233 gc_history_global gc_heap::bgc_data_global;
2235 GCEvent gc_heap::background_gc_done_event;
2237 GCEvent gc_heap::ee_proceed_event;
2239 bool gc_heap::gc_can_use_concurrent = false;
2241 bool gc_heap::temp_disable_concurrent_p = false;
2243 uint32_t gc_heap::cm_in_progress = FALSE;
2245 BOOL gc_heap::dont_restart_ee_p = FALSE;
2247 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2249 GCEvent gc_heap::bgc_threads_sync_event;
2251 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2253 BOOL gc_heap::do_concurrent_p = FALSE;
2255 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2257 BOOL gc_heap::alloc_wait_event_p = FALSE;
2259 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2261 VOLATILE(BOOL) gc_heap::gc_background_running = FALSE;
2262 #endif //BACKGROUND_GC
2264 #ifndef MULTIPLE_HEAPS
2265 #ifdef SPINLOCK_HISTORY
2266 int gc_heap::spinlock_info_index = 0;
2267 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2268 #endif //SPINLOCK_HISTORY
2270 uint32_t gc_heap::fgn_maxgen_percent = 0;
2271 size_t gc_heap::fgn_last_alloc = 0;
2273 int gc_heap::generation_skip_ratio = 100;
2275 uint64_t gc_heap::loh_alloc_since_cg = 0;
2277 BOOL gc_heap::elevation_requested = FALSE;
2279 BOOL gc_heap::last_gc_before_oom = FALSE;
2281 BOOL gc_heap::sufficient_gen0_space_p = FALSE;
2283 #ifdef BACKGROUND_GC
2284 uint8_t* gc_heap::background_saved_lowest_address = 0;
2285 uint8_t* gc_heap::background_saved_highest_address = 0;
2286 uint8_t* gc_heap::next_sweep_obj = 0;
2287 uint8_t* gc_heap::current_sweep_pos = 0;
2288 exclusive_sync* gc_heap::bgc_alloc_lock;
2289 #endif //BACKGROUND_GC
2291 oom_history gc_heap::oom_info;
2293 int gc_heap::oomhist_index_per_heap = 0;
2295 oom_history gc_heap::oomhist_per_heap[max_oom_history_count];
2297 fgm_history gc_heap::fgm_result;
2299 size_t gc_heap::allocated_since_last_gc = 0;
2301 BOOL gc_heap::ro_segments_in_range;
2303 size_t gc_heap::gen0_big_free_spaces = 0;
2305 uint8_t* gc_heap::ephemeral_low;
2307 uint8_t* gc_heap::ephemeral_high;
2309 uint8_t* gc_heap::lowest_address;
2311 uint8_t* gc_heap::highest_address;
2313 BOOL gc_heap::ephemeral_promotion;
2315 uint8_t* gc_heap::saved_ephemeral_plan_start[ephemeral_generation_count];
2316 size_t gc_heap::saved_ephemeral_plan_start_size[ephemeral_generation_count];
2318 short* gc_heap::brick_table;
2320 uint32_t* gc_heap::card_table;
2323 uint32_t* gc_heap::card_bundle_table;
2324 #endif //CARD_BUNDLE
2326 uint8_t* gc_heap::gc_low;
2328 uint8_t* gc_heap::gc_high;
2330 uint8_t* gc_heap::demotion_low;
2332 uint8_t* gc_heap::demotion_high;
2334 BOOL gc_heap::demote_gen1_p = TRUE;
2336 uint8_t* gc_heap::last_gen1_pin_end;
2338 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2340 size_t gc_heap::etw_allocation_running_amount[2];
2342 uint64_t gc_heap::total_alloc_bytes_soh = 0;
2344 uint64_t gc_heap::total_alloc_bytes_uoh = 0;
2346 int gc_heap::gc_policy = 0;
2348 size_t gc_heap::allocation_running_time;
2350 size_t gc_heap::allocation_running_amount;
2352 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2354 BOOL gc_heap::blocking_collection = FALSE;
2356 heap_segment* gc_heap::freeable_uoh_segment = 0;
2358 uint64_t gc_heap::time_bgc_last = 0;
2360 size_t gc_heap::mark_stack_tos = 0;
2362 size_t gc_heap::mark_stack_bos = 0;
2364 size_t gc_heap::mark_stack_array_length = 0;
2366 mark* gc_heap::mark_stack_array = 0;
2368 #if defined (_DEBUG) && defined (VERIFY_HEAP)
2369 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2370 #endif //_DEBUG && VERIFY_HEAP
2372 uint8_t* gc_heap::oldest_pinned_plug = 0;
2374 size_t gc_heap::num_pinned_objects = 0;
2376 #ifdef FEATURE_LOH_COMPACTION
2377 size_t gc_heap::loh_pinned_queue_tos = 0;
2379 size_t gc_heap::loh_pinned_queue_bos = 0;
2381 size_t gc_heap::loh_pinned_queue_length = 0;
2383 mark* gc_heap::loh_pinned_queue = 0;
2385 BOOL gc_heap::loh_compacted_p = FALSE;
2386 #endif //FEATURE_LOH_COMPACTION
2388 #ifdef BACKGROUND_GC
2390 EEThreadId gc_heap::bgc_thread_id;
2392 uint8_t* gc_heap::background_written_addresses [array_size+2];
2394 heap_segment* gc_heap::freeable_soh_segment = 0;
2396 size_t gc_heap::bgc_overflow_count = 0;
2398 size_t gc_heap::bgc_begin_loh_size = 0;
2399 size_t gc_heap::end_loh_size = 0;
2400 size_t gc_heap::bgc_begin_poh_size = 0;
2401 size_t gc_heap::end_poh_size = 0;
2403 #ifdef BGC_SERVO_TUNING
2404 uint64_t gc_heap::loh_a_no_bgc = 0;
2406 uint64_t gc_heap::loh_a_bgc_marking = 0;
2408 uint64_t gc_heap::loh_a_bgc_planning = 0;
2410 size_t gc_heap::bgc_maxgen_end_fl_size = 0;
2411 #endif //BGC_SERVO_TUNING
2413 uint32_t gc_heap::bgc_alloc_spin_uoh = 0;
2415 size_t gc_heap::bgc_loh_size_increased = 0;
2417 size_t gc_heap::bgc_poh_size_increased = 0;
2419 size_t gc_heap::background_soh_alloc_count = 0;
2421 size_t gc_heap::background_uoh_alloc_count = 0;
2423 uint8_t** gc_heap::background_mark_stack_tos = 0;
2425 uint8_t** gc_heap::background_mark_stack_array = 0;
2427 size_t gc_heap::background_mark_stack_array_length = 0;
2429 uint8_t* gc_heap::background_min_overflow_address =0;
2431 uint8_t* gc_heap::background_max_overflow_address =0;
2433 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2435 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2437 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2439 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2441 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2443 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2445 Thread* gc_heap::bgc_thread = 0;
2447 BOOL gc_heap::expanded_in_fgc = FALSE;
2449 uint8_t** gc_heap::c_mark_list = 0;
2451 size_t gc_heap::c_mark_list_length = 0;
2453 size_t gc_heap::c_mark_list_index = 0;
2455 gc_history_per_heap gc_heap::bgc_data_per_heap;
2457 BOOL gc_heap::bgc_thread_running;
2459 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2461 #endif //BACKGROUND_GC
2464 uint8_t** gc_heap::mark_list;
2465 uint8_t** gc_heap::mark_list_index;
2466 uint8_t** gc_heap::mark_list_end;
2470 snoop_stats_data gc_heap::snoop_stat;
2471 #endif //SNOOP_STATS
2473 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2475 uint8_t* gc_heap::max_overflow_address = 0;
2477 uint8_t* gc_heap::shigh = 0;
2479 uint8_t* gc_heap::slow = MAX_PTR;
2481 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2483 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2485 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2487 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2489 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2491 BOOL gc_heap::use_bestfit = FALSE;
2493 uint8_t* gc_heap::bestfit_first_pin = 0;
2495 BOOL gc_heap::commit_end_of_seg = FALSE;
2497 size_t gc_heap::max_free_space_items = 0;
2499 size_t gc_heap::free_space_buckets = 0;
2501 size_t gc_heap::free_space_items = 0;
2503 int gc_heap::trimmed_free_space_index = 0;
2505 size_t gc_heap::total_ephemeral_plugs = 0;
2507 seg_free_spaces* gc_heap::bestfit_seg = 0;
2509 size_t gc_heap::total_ephemeral_size = 0;
2513 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2515 uint8_t** gc_heap::internal_root_array = 0;
2517 size_t gc_heap::internal_root_array_index = 0;
2519 BOOL gc_heap::heap_analyze_success = TRUE;
2521 uint8_t* gc_heap::current_obj = 0;
2522 size_t gc_heap::current_obj_size = 0;
2524 #endif //HEAP_ANALYZE
2526 #ifdef GC_CONFIG_DRIVEN
2527 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2528 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2529 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2530 #endif //GC_CONFIG_DRIVEN
2531 #endif //MULTIPLE_HEAPS
2533 no_gc_region_info gc_heap::current_no_gc_region_info;
2534 BOOL gc_heap::proceed_with_gc_p = FALSE;
2535 GCSpinLock gc_heap::gc_lock;
2537 #ifdef BGC_SERVO_TUNING
2538 uint64_t gc_heap::total_loh_a_last_bgc = 0;
2539 #endif //BGC_SERVO_TUNING
2541 size_t gc_heap::eph_gen_starts_size = 0;
2542 heap_segment* gc_heap::segment_standby_list;
2543 bool gc_heap::use_large_pages_p = 0;
2544 #ifdef HEAP_BALANCE_INSTRUMENTATION
2545 size_t gc_heap::last_gc_end_time_us = 0;
2546 #endif //HEAP_BALANCE_INSTRUMENTATION
2547 size_t gc_heap::min_segment_size = 0;
2548 size_t gc_heap::min_segment_size_shr = 0;
2549 size_t gc_heap::soh_segment_size = 0;
2550 size_t gc_heap::min_uoh_segment_size = 0;
2551 size_t gc_heap::segment_info_size = 0;
2553 #ifdef GC_CONFIG_DRIVEN
2554 size_t gc_heap::compact_or_sweep_gcs[2];
2555 #endif //GC_CONFIG_DRIVEN
2557 #ifdef FEATURE_LOH_COMPACTION
2558 BOOL gc_heap::loh_compaction_always_p = FALSE;
2559 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2560 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2562 #endif //FEATURE_LOH_COMPACTION
2564 GCEvent gc_heap::full_gc_approach_event;
2566 GCEvent gc_heap::full_gc_end_event;
2568 uint32_t gc_heap::fgn_loh_percent = 0;
2570 #ifdef BACKGROUND_GC
2571 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2572 #endif //BACKGROUND_GC
2574 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2576 size_t gc_heap::full_gc_counts[gc_type_max];
2578 bool gc_heap::maxgen_size_inc_p = false;
2580 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2582 // Provisional mode related stuff.
2583 bool gc_heap::provisional_mode_triggered = false;
2584 bool gc_heap::pm_trigger_full_gc = false;
2585 size_t gc_heap::provisional_triggered_gc_count = 0;
2586 size_t gc_heap::provisional_off_gc_count = 0;
2587 size_t gc_heap::num_provisional_triggered = 0;
2588 bool gc_heap::pm_stress_on = false;
2591 BOOL gc_heap::heap_analyze_enabled = FALSE;
2592 #endif //HEAP_ANALYZE
2594 #ifndef MULTIPLE_HEAPS
2596 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2597 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2598 alloc_list gc_heap::poh_alloc_list [NUM_POH_ALIST-1];
2600 dynamic_data gc_heap::dynamic_data_table [total_generation_count];
2601 gc_history_per_heap gc_heap::gc_data_per_heap;
2602 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2604 uint8_t* gc_heap::alloc_allocated = 0;
2606 size_t gc_heap::allocation_quantum = CLR_SIZE;
2608 GCSpinLock gc_heap::more_space_lock_soh;
2609 GCSpinLock gc_heap::more_space_lock_uoh;
2611 VOLATILE(int32_t) gc_heap::uoh_alloc_thread_count = 0;
2613 #ifdef SYNCHRONIZATION_STATS
2614 unsigned int gc_heap::good_suspension = 0;
2615 unsigned int gc_heap::bad_suspension = 0;
2616 uint64_t gc_heap::total_msl_acquire = 0;
2617 unsigned int gc_heap::num_msl_acquired = 0;
2618 unsigned int gc_heap::num_high_msl_acquire = 0;
2619 unsigned int gc_heap::num_low_msl_acquire = 0;
2620 #endif //SYNCHRONIZATION_STATS
2622 size_t gc_heap::alloc_contexts_used = 0;
2623 size_t gc_heap::soh_allocation_no_gc = 0;
2624 size_t gc_heap::loh_allocation_no_gc = 0;
2625 bool gc_heap::no_gc_oom_p = false;
2626 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2628 #endif //MULTIPLE_HEAPS
2630 #ifndef MULTIPLE_HEAPS
2632 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2634 int gc_heap::gen0_must_clear_bricks = 0;
2636 #ifdef FEATURE_PREMORTEM_FINALIZATION
2637 CFinalize* gc_heap::finalize_queue = 0;
2638 #endif // FEATURE_PREMORTEM_FINALIZATION
2640 #ifdef FEATURE_CARD_MARKING_STEALING
2641 VOLATILE(uint32_t) gc_heap::card_mark_chunk_index_soh;
2642 VOLATILE(bool) gc_heap::card_mark_done_soh;
2643 VOLATILE(uint32_t) gc_heap::card_mark_chunk_index_loh;
2644 VOLATILE(uint32_t) gc_heap::card_mark_chunk_index_poh;
2645 VOLATILE(bool) gc_heap::card_mark_done_uoh;
2646 #endif // FEATURE_CARD_MARKING_STEALING
2648 generation gc_heap::generation_table [total_generation_count];
2650 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2652 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2654 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2656 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2658 #endif // MULTIPLE_HEAPS
2660 /* end of per heap static initialization */
2662 /* end of static initialization */
2664 #ifndef DACCESS_COMPILE
2666 void gen_to_condemn_tuning::print (int heap_num)
2669 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2670 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2671 gc_condemn_reason_gen r_gen;
2672 for (int i = 0; i < gcrg_max; i++)
2674 r_gen = (gc_condemn_reason_gen)(i);
2675 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2677 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2679 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2680 gc_condemn_reason_condition r_condition;
2681 for (int i = 0; i < gcrc_max; i++)
2683 r_condition = (gc_condemn_reason_condition)(i);
2684 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2687 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2689 UNREFERENCED_PARAMETER(heap_num);
2693 void gc_generation_data::print (int heap_num, int gen_num)
2695 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2696 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2699 free_list_space_before, free_obj_space_before,
2701 free_list_space_after, free_obj_space_after,
2702 in, pinned_surv, npinned_surv,
2705 UNREFERENCED_PARAMETER(heap_num);
2706 UNREFERENCED_PARAMETER(gen_num);
2707 #endif //SIMPLE_DPRINTF && DT_LOG
2710 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2712 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2714 *mechanism |= mechanism_mask;
2715 *mechanism |= (1 << value);
2718 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
2719 dprintf (DT_LOG_0, ("setting %s: %s",
2721 (descr->descr)[value]));
2725 void gc_history_per_heap::print()
2727 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2728 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
2730 gen_data[i].print (heap_index, i);
2733 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
2734 maxgen_size_info.free_list_allocated,
2735 maxgen_size_info.free_list_rejected,
2736 maxgen_size_info.end_seg_allocated,
2737 maxgen_size_info.condemned_allocated,
2738 maxgen_size_info.pinned_allocated,
2739 maxgen_size_info.pinned_allocated_advance,
2740 maxgen_size_info.running_free_list_efficiency,
2741 extra_gen0_committed));
2744 gc_mechanism_descr* descr = 0;
2746 for (int i = 0; i < max_mechanism_per_heap; i++)
2748 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
2752 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
2753 dprintf (DT_LOG_0, ("[%2d]%s%s",
2756 (descr->descr)[mechanism]));
2759 #endif //SIMPLE_DPRINTF && DT_LOG
2762 void gc_history_global::print()
2765 char str_settings[64];
2766 memset (str_settings, '|', sizeof (char) * 64);
2767 str_settings[max_global_mechanisms_count*2] = 0;
2769 for (int i = 0; i < max_global_mechanisms_count; i++)
2771 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
2774 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
2776 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
2777 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
2778 condemned_generation,
2779 str_gc_reasons[reason],
2780 str_gc_pause_modes[pause_mode],
2781 final_youngest_desired,
2782 gen0_reduction_count,
2787 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
2789 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
2790 FIRE_EVENT(GCPerHeapHistory_V3,
2791 (void *)(maxgen_size_info->free_list_allocated),
2792 (void *)(maxgen_size_info->free_list_rejected),
2793 (void *)(maxgen_size_info->end_seg_allocated),
2794 (void *)(maxgen_size_info->condemned_allocated),
2795 (void *)(maxgen_size_info->pinned_allocated),
2796 (void *)(maxgen_size_info->pinned_allocated_advance),
2797 maxgen_size_info->running_free_list_efficiency,
2798 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
2799 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
2800 current_gc_data_per_heap->mechanisms[gc_heap_compact],
2801 current_gc_data_per_heap->mechanisms[gc_heap_expand],
2802 current_gc_data_per_heap->heap_index,
2803 (void *)(current_gc_data_per_heap->extra_gen0_committed),
2804 total_generation_count,
2805 (uint32_t)(sizeof (gc_generation_data)),
2806 (void *)&(current_gc_data_per_heap->gen_data[0]));
2808 current_gc_data_per_heap->print();
2809 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
2812 void gc_heap::fire_pevents()
2814 gc_history_global* current_gc_data_global = get_gc_data_global();
2816 settings.record (current_gc_data_global);
2817 current_gc_data_global->print();
2819 FIRE_EVENT(GCGlobalHeapHistory_V3,
2820 current_gc_data_global->final_youngest_desired,
2821 current_gc_data_global->num_heaps,
2822 current_gc_data_global->condemned_generation,
2823 current_gc_data_global->gen0_reduction_count,
2824 current_gc_data_global->reason,
2825 current_gc_data_global->global_mechanisms_p,
2826 current_gc_data_global->pause_mode,
2827 current_gc_data_global->mem_pressure,
2828 current_gc_data_global->gen_to_condemn_reasons.get_reasons0(),
2829 current_gc_data_global->gen_to_condemn_reasons.get_reasons1());
2831 #ifdef MULTIPLE_HEAPS
2832 for (int i = 0; i < gc_heap::n_heaps; i++)
2834 gc_heap* hp = gc_heap::g_heaps[i];
2835 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
2836 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
2839 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
2840 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
2845 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
2851 case tuning_deciding_condemned_gen:
2852 case tuning_deciding_compaction:
2853 case tuning_deciding_expansion:
2854 case tuning_deciding_full_gc:
2856 ret = (!ephemeral_gen_fit_p (tp));
2859 case tuning_deciding_promote_ephemeral:
2861 size_t new_gen0size = approximate_new_allocation();
2862 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
2864 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
2865 heap_number, plan_ephemeral_size, new_gen0size));
2866 // If we were in no_gc_region we could have allocated a larger than normal segment,
2867 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
2868 // ephemeral generations there, do an ephemeral promotion.
2869 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
2880 gc_heap::dt_high_frag_p (gc_tuning_point tp,
2888 case tuning_deciding_condemned_gen:
2890 dynamic_data* dd = dynamic_data_of (gen_number);
2891 float fragmentation_burden = 0;
2895 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
2896 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
2897 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
2901 #ifndef MULTIPLE_HEAPS
2902 if (gen_number == max_generation)
2904 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
2905 if (frag_ratio > 0.65)
2907 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
2911 #endif //!MULTIPLE_HEAPS
2912 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
2913 ret = (fr > dd_fragmentation_limit(dd));
2916 fragmentation_burden = (float)fr / generation_size (gen_number);
2917 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
2919 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
2920 heap_number, gen_number, dd_fragmentation (dd),
2921 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
2922 fr, (int)(fragmentation_burden*100)));
2934 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
2940 case tuning_deciding_condemned_gen:
2942 if (gen_number == max_generation)
2944 size_t est_maxgen_free = estimated_reclaim (gen_number);
2946 uint32_t num_heaps = 1;
2947 #ifdef MULTIPLE_HEAPS
2948 num_heaps = gc_heap::n_heaps;
2949 #endif //MULTIPLE_HEAPS
2951 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
2952 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
2953 ret = (est_maxgen_free >= min_frag_th);
2969 // DTREVIEW: Right now we only estimate gen2 fragmentation.
2970 // on 64-bit though we should consider gen1 or even gen0 fragmentation as
2973 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
2979 case tuning_deciding_condemned_gen:
2981 if (gen_number == max_generation)
2983 dynamic_data* dd = dynamic_data_of (gen_number);
2984 float est_frag_ratio = 0;
2985 if (dd_current_size (dd) == 0)
2989 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
2995 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
2998 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
2999 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3002 dd_current_size (dd),
3003 dd_fragmentation (dd),
3004 (int)(est_frag_ratio*100),
3007 uint32_t num_heaps = 1;
3009 #ifdef MULTIPLE_HEAPS
3010 num_heaps = gc_heap::n_heaps;
3011 #endif //MULTIPLE_HEAPS
3012 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3013 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3014 ret = (est_frag >= min_frag_th);
3031 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3037 case tuning_deciding_condemned_gen:
3039 /* promote into max-generation if the card table has too many
3040 * generation faults besides the n -> 0
3042 ret = (generation_skip_ratio < 30);
3054 in_range_for_segment(uint8_t* add, heap_segment* seg)
3056 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3059 #ifdef FEATURE_BASICFREEZE
3060 // The array we allocate is organized as follows:
3061 // 0th element is the address of the last array we allocated.
3062 // starting from the 1st element are the segment addresses, that's
3063 // what buckets() returns.
3076 bk* buckets() { return (slots + 1); }
3077 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3080 static sorted_table* make_sorted_table ();
3081 BOOL insert (uint8_t* add, size_t val);;
3082 size_t lookup (uint8_t*& add);
3083 void remove (uint8_t* add);
3085 void delete_sorted_table();
3086 void delete_old_slots();
3087 void enqueue_old_slot(bk* sl);
3088 BOOL ensure_space_for_insert();
3092 sorted_table::make_sorted_table ()
3096 // allocate one more bk to store the older slot address.
3097 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3101 res->slots = (bk*)(res + 1);
3108 sorted_table::delete_sorted_table()
3110 if (slots != (bk*)(this+1))
3118 sorted_table::delete_old_slots()
3120 uint8_t* sl = (uint8_t*)old_slots;
3124 sl = last_slot ((bk*)sl);
3130 sorted_table::enqueue_old_slot(bk* sl)
3132 last_slot (sl) = (uint8_t*)old_slots;
3138 sorted_table::lookup (uint8_t*& add)
3140 ptrdiff_t high = (count-1);
3144 bk* buck = buckets();
3147 mid = ((low + high)/2);
3149 if (buck[ti].add > add)
3151 if ((ti > 0) && (buck[ti-1].add <= add))
3153 add = buck[ti-1].add;
3154 return buck[ti - 1].val;
3160 if (buck[ti+1].add > add)
3163 return buck[ti].val;
3173 sorted_table::ensure_space_for_insert()
3177 size = (size * 3)/2;
3178 assert((size * sizeof (bk)) > 0);
3179 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3184 last_slot (res) = 0;
3185 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3186 bk* last_old_slots = slots;
3188 if (last_old_slots != (bk*)(this + 1))
3189 enqueue_old_slot (last_old_slots);
3195 sorted_table::insert (uint8_t* add, size_t val)
3197 //grow if no more room
3198 assert (count < size);
3201 ptrdiff_t high = (count-1);
3205 bk* buck = buckets();
3208 mid = ((low + high)/2);
3210 if (buck[ti].add > add)
3212 if ((ti == 0) || (buck[ti-1].add <= add))
3214 // found insertion point
3215 for (ptrdiff_t k = count; k > ti;k--)
3217 buck [k] = buck [k-1];
3228 if (buck[ti+1].add > add)
3230 //found the insertion point
3231 for (ptrdiff_t k = count; k > ti+1;k--)
3233 buck [k] = buck [k-1];
3235 buck[ti+1].add = add;
3236 buck[ti+1].val = val;
3248 sorted_table::remove (uint8_t* add)
3250 ptrdiff_t high = (count-1);
3254 bk* buck = buckets();
3257 mid = ((low + high)/2);
3259 if (buck[ti].add > add)
3261 if (buck[ti-1].add <= add)
3263 for (ptrdiff_t k = ti; k < count; k++)
3264 buck[k-1] = buck[k];
3272 if (buck[ti+1].add > add)
3274 for (ptrdiff_t k = ti+1; k < count; k++)
3275 buck[k-1] = buck[k];
3286 sorted_table::clear()
3289 buckets()[0].add = MAX_PTR;
3291 #endif //FEATURE_BASICFREEZE
3294 uint8_t* align_on_segment (uint8_t* add)
3296 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3300 uint8_t* align_lower_segment (uint8_t* add)
3302 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3305 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3307 from = align_lower_segment (from);
3308 end = align_on_segment (end);
3309 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3310 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3313 // for seg_mapping_table we want it to start from a pointer sized address.
3315 size_t align_for_seg_mapping_table (size_t size)
3317 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3321 size_t seg_mapping_word_of (uint8_t* add)
3323 return (size_t)add >> gc_heap::min_segment_size_shr;
3326 #ifdef FEATURE_BASICFREEZE
3328 size_t ro_seg_begin_index (heap_segment* seg)
3330 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3331 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3336 size_t ro_seg_end_index (heap_segment* seg)
3338 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3339 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3343 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3345 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3348 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3349 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3352 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3354 UNREFERENCED_PARAMETER(seg);
3356 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3357 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3358 // remove the flag if none lands in this range.
3362 heap_segment* ro_segment_lookup (uint8_t* o)
3364 uint8_t* ro_seg_start = o;
3365 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3367 if (ro_seg_start && in_range_for_segment (o, seg))
3373 #endif //FEATURE_BASICFREEZE
3375 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3377 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3378 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3379 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3380 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3381 seg_mapping* end_entry = &seg_mapping_table[end_index];
3383 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3384 seg, begin_index, heap_segment_reserved (seg), end_index));
3386 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3387 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3388 end_index, (seg_mapping_table[end_index].boundary + 1)));
3390 #ifdef MULTIPLE_HEAPS
3391 #ifdef SIMPLE_DPRINTF
3392 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3393 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3394 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3395 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3396 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3397 #endif //SIMPLE_DPRINTF
3398 assert (end_entry->boundary == 0);
3399 assert (end_entry->h0 == 0);
3401 assert (begin_entry->h1 == 0);
3402 begin_entry->h1 = hp;
3404 UNREFERENCED_PARAMETER(hp);
3405 #endif //MULTIPLE_HEAPS
3407 end_entry->boundary = (uint8_t*)seg_end;
3409 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3410 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3411 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3412 end_entry->seg0 = seg;
3414 // for every entry inbetween we need to set its heap too.
3415 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3417 assert (seg_mapping_table[entry_index].boundary == 0);
3418 #ifdef MULTIPLE_HEAPS
3419 assert (seg_mapping_table[entry_index].h0 == 0);
3420 seg_mapping_table[entry_index].h1 = hp;
3421 #endif //MULTIPLE_HEAPS
3422 seg_mapping_table[entry_index].seg1 = seg;
3425 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3426 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3427 end_index, (seg_mapping_table[end_index].boundary + 1)));
3428 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3429 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3430 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3431 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3432 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3433 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3434 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3437 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3439 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3440 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3441 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3442 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3443 seg_mapping* end_entry = &seg_mapping_table[end_index];
3444 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3445 seg, begin_index, heap_segment_reserved (seg), end_index));
3447 assert (end_entry->boundary == (uint8_t*)seg_end);
3448 end_entry->boundary = 0;
3450 #ifdef MULTIPLE_HEAPS
3451 gc_heap* hp = heap_segment_heap (seg);
3452 assert (end_entry->h0 == hp);
3454 assert (begin_entry->h1 == hp);
3455 begin_entry->h1 = 0;
3456 #endif //MULTIPLE_HEAPS
3458 assert (begin_entry->seg1 != 0);
3459 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3460 end_entry->seg0 = 0;
3462 // for every entry inbetween we need to reset its heap too.
3463 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3465 assert (seg_mapping_table[entry_index].boundary == 0);
3466 #ifdef MULTIPLE_HEAPS
3467 assert (seg_mapping_table[entry_index].h0 == 0);
3468 assert (seg_mapping_table[entry_index].h1 == hp);
3469 seg_mapping_table[entry_index].h1 = 0;
3470 #endif //MULTIPLE_HEAPS
3471 seg_mapping_table[entry_index].seg1 = 0;
3474 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3475 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3476 end_index, (seg_mapping_table[end_index].boundary + 1)));
3477 #ifdef MULTIPLE_HEAPS
3478 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3479 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3480 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3481 #endif //MULTIPLE_HEAPS
3484 #ifdef MULTIPLE_HEAPS
3486 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3488 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3489 seg_mapping* entry = &seg_mapping_table[index];
3491 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3493 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundary: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3494 o, index, (entry->boundary + 1),
3495 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3496 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3499 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3500 #ifdef FEATURE_BASICFREEZE
3501 if ((size_t)seg & ro_in_entry)
3502 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3503 #endif //FEATURE_BASICFREEZE
3508 if (in_range_for_segment (o, seg))
3510 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3514 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3515 seg, (uint8_t*)heap_segment_allocated (seg), o));
3520 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3528 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3530 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3533 return seg_mapping_table_heap_of_worker (o);
3536 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3538 #ifdef FEATURE_BASICFREEZE
3539 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3541 #endif //FEATURE_BASICFREEZE
3543 return seg_mapping_table_heap_of_worker (o);
3545 #endif //MULTIPLE_HEAPS
3547 // Only returns a valid seg if we can actually find o on the seg.
3548 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3550 #ifdef FEATURE_BASICFREEZE
3551 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3552 return ro_segment_lookup (o);
3553 #endif //FEATURE_BASICFREEZE
3555 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3556 seg_mapping* entry = &seg_mapping_table[index];
3558 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundary: %Ix, seg0: %Ix, seg1: %Ix",
3559 o, index, (entry->boundary + 1),
3560 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3562 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3563 #ifdef FEATURE_BASICFREEZE
3564 if ((size_t)seg & ro_in_entry)
3565 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3566 #endif //FEATURE_BASICFREEZE
3570 if (in_range_for_segment (o, seg))
3572 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3576 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3577 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3583 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3586 #ifdef FEATURE_BASICFREEZE
3587 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3588 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3589 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3590 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3591 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3592 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3595 seg = ro_segment_lookup (o);
3596 if (seg && !in_range_for_segment (o, seg))
3599 #endif //FEATURE_BASICFREEZE
3604 size_t gcard_of ( uint8_t*);
3606 #define GC_MARKED (size_t)0x1
3607 #define slot(i, j) ((uint8_t**)(i))[(j)+1]
3609 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3611 class CObjectHeader : public Object
3615 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3616 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3617 // by Redhawk's version of Object.
3618 uint32_t GetNumComponents()
3620 return ((ArrayBase *)this)->GetNumComponents();
3623 void Validate(BOOL bDeep=TRUE)
3625 MethodTable * pMT = GetMethodTable();
3627 _ASSERTE(pMT->SanityCheck());
3629 bool noRangeChecks =
3630 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3632 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3635 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3636 if (!fSmallObjectHeapPtr)
3637 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3639 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3642 #ifdef FEATURE_STRUCTALIGN
3643 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3644 #endif // FEATURE_STRUCTALIGN
3646 #ifdef FEATURE_64BIT_ALIGNMENT
3647 if (pMT->RequiresAlign8())
3649 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3651 #endif // FEATURE_64BIT_ALIGNMENT
3654 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3655 g_theGCHeap->ValidateObjectMember(this);
3657 if (fSmallObjectHeapPtr)
3659 #ifdef FEATURE_BASICFREEZE
3660 _ASSERTE(!g_theGCHeap->IsLargeObject(this) || g_theGCHeap->IsInFrozenSegment(this));
3662 _ASSERTE(!g_theGCHeap->IsLargeObject(this));
3667 void ValidateHeap(BOOL bDeep)
3672 #endif //FEATURE_REDHAWK || BUILD_AS_STANDALONE
3676 // Header Status Information
3679 MethodTable *GetMethodTable() const
3681 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
3686 _ASSERTE(RawGetMethodTable());
3687 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
3690 BOOL IsMarked() const
3692 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
3697 assert (!(gc_heap::settings.concurrent));
3698 GetHeader()->SetGCBit();
3701 BOOL IsPinned() const
3703 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
3708 RawSetMethodTable( GetMethodTable() );
3711 CGCDesc *GetSlotMap ()
3713 assert (GetMethodTable()->ContainsPointers());
3714 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
3717 void SetFree(size_t size)
3719 assert (size >= free_object_base_size);
3721 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
3722 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
3724 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
3726 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
3727 *numComponentsPtr = size - free_object_base_size;
3729 //This introduces a bug in the free list management.
3730 //((void**) this)[-1] = 0; // clear the sync block,
3731 assert (*numComponentsPtr >= 0);
3732 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
3733 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
3734 #endif //VERIFY_HEAP
3739 size_t size = free_object_base_size - plug_skew;
3741 // since we only need to clear 2 ptr size, we do it manually
3742 PTR_PTR m = (PTR_PTR) this;
3743 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
3747 BOOL IsFree () const
3749 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
3752 #ifdef FEATURE_STRUCTALIGN
3753 int GetRequiredAlignment () const
3755 return GetMethodTable()->GetRequiredAlignment();
3757 #endif // FEATURE_STRUCTALIGN
3759 BOOL ContainsPointers() const
3761 return GetMethodTable()->ContainsPointers();
3764 #ifdef COLLECTIBLE_CLASS
3765 BOOL Collectible() const
3767 return GetMethodTable()->Collectible();
3770 FORCEINLINE BOOL ContainsPointersOrCollectible() const
3772 MethodTable *pMethodTable = GetMethodTable();
3773 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
3775 #endif //COLLECTIBLE_CLASS
3777 Object* GetObjectBase() const
3779 return (Object*) this;
3783 #define header(i) ((CObjectHeader*)(i))
3785 #define free_list_slot(x) ((uint8_t**)(x))[2]
3786 #define free_list_undo(x) ((uint8_t**)(x))[-1]
3787 #define UNDO_EMPTY ((uint8_t*)1)
3791 void set_plug_padded (uint8_t* node)
3793 header(node)->SetMarked();
3796 void clear_plug_padded (uint8_t* node)
3798 header(node)->ClearMarked();
3801 BOOL is_plug_padded (uint8_t* node)
3803 return header(node)->IsMarked();
3806 inline void set_plug_padded (uint8_t* node){}
3807 inline void clear_plug_padded (uint8_t* node){}
3809 BOOL is_plug_padded (uint8_t* node){return FALSE;}
3810 #endif //SHORT_PLUGS
3813 inline size_t unused_array_size(uint8_t * p)
3815 assert(((CObjectHeader*)p)->IsFree());
3817 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
3818 return free_object_base_size + *numComponentsPtr;
3821 heap_segment* heap_segment_rw (heap_segment* ns)
3823 if ((ns == 0) || !heap_segment_read_only_p (ns))
3831 ns = heap_segment_next (ns);
3832 } while ((ns != 0) && heap_segment_read_only_p (ns));
3837 //returns the next non ro segment.
3838 heap_segment* heap_segment_next_rw (heap_segment* seg)
3840 heap_segment* ns = heap_segment_next (seg);
3841 return heap_segment_rw (ns);
3844 // returns the segment before seg.
3845 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
3847 assert (begin != 0);
3848 heap_segment* prev = begin;
3849 heap_segment* current = heap_segment_next_rw (begin);
3851 while (current && current != seg)
3854 current = heap_segment_next_rw (current);
3867 // returns the segment before seg.
3868 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
3870 assert (begin != 0);
3871 heap_segment* prev = begin;
3872 heap_segment* current = heap_segment_next (begin);
3874 while (current && current != seg)
3877 current = heap_segment_next (current);
3890 heap_segment* heap_segment_in_range (heap_segment* ns)
3892 if ((ns == 0) || heap_segment_in_range_p (ns))
3900 ns = heap_segment_next (ns);
3901 } while ((ns != 0) && !heap_segment_in_range_p (ns));
3906 heap_segment* heap_segment_next_in_range (heap_segment* seg)
3908 heap_segment* ns = heap_segment_next (seg);
3909 return heap_segment_in_range (ns);
3914 uint8_t* memory_base;
3917 struct initial_memory_details
3919 imemory_data *initial_memory;
3920 imemory_data *initial_normal_heap; // points into initial_memory_array
3921 imemory_data *initial_large_heap; // points into initial_memory_array
3922 imemory_data *initial_pinned_heap; // points into initial_memory_array
3924 size_t block_size_normal;
3925 size_t block_size_large;
3926 size_t block_size_pinned;
3928 int block_count; // # of blocks in each
3929 int current_block_normal;
3930 int current_block_large;
3931 int current_block_pinned;
3938 ALLATONCE_SEPARATED_POH
3941 size_t allocation_pattern;
3943 size_t block_size(int i)
3945 switch (i / block_count)
3947 case 0: return block_size_normal;
3948 case 1: return block_size_large;
3949 case 2: return block_size_pinned;
3950 default: __UNREACHABLE();
3954 void* get_initial_memory (int gen, int h_number)
3960 case soh_gen2: return initial_normal_heap[h_number].memory_base;
3961 case loh_generation: return initial_large_heap[h_number].memory_base;
3962 case poh_generation: return initial_pinned_heap[h_number].memory_base;
3963 default: __UNREACHABLE();
3967 size_t get_initial_size (int gen)
3973 case soh_gen2: return block_size_normal;
3974 case loh_generation: return block_size_large;
3975 case poh_generation: return block_size_pinned;
3976 default: __UNREACHABLE();
3982 initial_memory_details memory_details;
3984 BOOL gc_heap::reserve_initial_memory (size_t normal_size, size_t large_size, size_t pinned_size, int num_heaps, bool use_large_pages_p, bool separated_poh_p)
3986 BOOL reserve_success = FALSE;
3988 // should only be called once
3989 assert (memory_details.initial_memory == 0);
3991 // soh + loh + poh segments * num_heaps
3992 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps * (total_generation_count - ephemeral_generation_count)];
3993 if (memory_details.initial_memory == 0)
3995 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps * (total_generation_count - ephemeral_generation_count) * sizeof (imemory_data)));
3999 memory_details.initial_normal_heap = memory_details.initial_memory;
4000 memory_details.initial_large_heap = memory_details.initial_normal_heap + num_heaps;
4001 memory_details.initial_pinned_heap = memory_details.initial_large_heap + num_heaps;
4002 memory_details.block_size_normal = normal_size;
4003 memory_details.block_size_large = large_size;
4004 memory_details.block_size_pinned = pinned_size;
4006 memory_details.block_count = num_heaps;
4008 memory_details.current_block_normal = 0;
4009 memory_details.current_block_large = 0;
4010 memory_details.current_block_pinned = 0;
4012 g_gc_lowest_address = MAX_PTR;
4013 g_gc_highest_address = 0;
4015 if (((size_t)MAX_PTR - large_size) < normal_size)
4017 // we are already overflowing with just one heap.
4018 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4022 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size + pinned_size))
4024 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4028 size_t temp_pinned_size = (separated_poh_p ? 0 : pinned_size);
4029 size_t separate_pinned_size = memory_details.block_count * pinned_size;
4030 size_t requestedMemory = memory_details.block_count * (normal_size + large_size + temp_pinned_size);
4032 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory, use_large_pages_p);
4033 uint8_t* separated_poh_block = nullptr;
4034 if (allatonce_block && separated_poh_p)
4036 separated_poh_block = (uint8_t*)virtual_alloc (separate_pinned_size, false);
4037 if (!separated_poh_block)
4039 virtual_free (allatonce_block, requestedMemory);
4040 allatonce_block = nullptr;
4043 if (allatonce_block)
4045 if (separated_poh_p)
4047 g_gc_lowest_address = min (allatonce_block, separated_poh_block);
4048 g_gc_highest_address = max ((allatonce_block + requestedMemory), (separated_poh_block + separate_pinned_size));
4049 memory_details.allocation_pattern = initial_memory_details::ALLATONCE_SEPARATED_POH;
4053 g_gc_lowest_address = allatonce_block;
4054 g_gc_highest_address = allatonce_block + requestedMemory;
4055 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4058 for (int i = 0; i < memory_details.block_count; i++)
4060 memory_details.initial_normal_heap[i].memory_base = allatonce_block +
4062 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4063 (memory_details.block_count * normal_size) + (i * large_size);
4064 if (separated_poh_p)
4066 memory_details.initial_pinned_heap[i].memory_base = separated_poh_block +
4071 memory_details.initial_pinned_heap[i].memory_base = allatonce_block +
4072 (memory_details.block_count * (normal_size + large_size)) + (i * pinned_size);
4075 reserve_success = TRUE;
4080 // try to allocate 3 blocks
4081 uint8_t* b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size, use_large_pages_p);
4082 uint8_t* b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size, use_large_pages_p);
4083 uint8_t* b3 = (uint8_t*)virtual_alloc (memory_details.block_count * pinned_size, use_large_pages_p && !separated_poh_p);
4087 memory_details.allocation_pattern = initial_memory_details::EACH_GENERATION;
4088 g_gc_lowest_address = min (b1, min(b2, b3));
4089 g_gc_highest_address = max (b1 + memory_details.block_count * normal_size,
4090 max (b2 + memory_details.block_count * large_size,
4091 b3 + memory_details.block_count * pinned_size));
4093 for (int i = 0; i < memory_details.block_count; i++)
4095 memory_details.initial_normal_heap[i].memory_base = b1 + (i * normal_size);
4096 memory_details.initial_large_heap[i].memory_base = b2 + (i * large_size);
4097 memory_details.initial_pinned_heap[i].memory_base = b3 + (i * pinned_size);
4100 reserve_success = TRUE;
4104 // allocation failed, we'll go on to try allocating each block.
4105 // We could preserve the b1 alloc, but code complexity increases
4107 virtual_free (b1, memory_details.block_count * normal_size);
4109 virtual_free (b2, memory_details.block_count * large_size);
4111 virtual_free (b3, memory_details.block_count * pinned_size);
4114 if ((b2 == NULL) && (memory_details.block_count > 1))
4116 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4118 imemory_data* current_block = memory_details.initial_memory;
4119 for (int i = 0; i < (memory_details.block_count * (total_generation_count - ephemeral_generation_count)); i++, current_block++)
4121 size_t block_size = memory_details.block_size (i);
4122 current_block->memory_base =
4123 (uint8_t*)virtual_alloc (block_size, use_large_pages_p);
4124 if (current_block->memory_base == 0)
4126 // Free the blocks that we've allocated so far
4127 current_block = memory_details.initial_memory;
4128 for (int j = 0; j < i; j++, current_block++) {
4129 if (current_block->memory_base != 0) {
4130 block_size = memory_details.block_size (i);
4131 virtual_free (current_block->memory_base, block_size);
4134 reserve_success = FALSE;
4139 if (current_block->memory_base < g_gc_lowest_address)
4140 g_gc_lowest_address = current_block->memory_base;
4141 if (((uint8_t*)current_block->memory_base + block_size) > g_gc_highest_address)
4142 g_gc_highest_address = (current_block->memory_base + block_size);
4144 reserve_success = TRUE;
4149 return reserve_success;
4152 void gc_heap::destroy_initial_memory()
4154 if (memory_details.initial_memory != NULL)
4156 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4158 virtual_free(memory_details.initial_memory[0].memory_base,
4159 memory_details.block_count*(memory_details.block_size_normal +
4160 memory_details.block_size_large + memory_details.block_size_pinned));
4162 else if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE_SEPARATED_POH)
4164 virtual_free(memory_details.initial_memory[0].memory_base,
4165 memory_details.block_count*(memory_details.block_size_normal +
4166 memory_details.block_size_large));
4167 virtual_free(memory_details.initial_pinned_heap[0].memory_base,
4168 memory_details.block_count*(memory_details.block_size_pinned));
4170 else if (memory_details.allocation_pattern == initial_memory_details::EACH_GENERATION)
4172 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4173 memory_details.block_count*memory_details.block_size_normal);
4175 virtual_free (memory_details.initial_large_heap[0].memory_base,
4176 memory_details.block_count*memory_details.block_size_large);
4178 virtual_free (memory_details.initial_pinned_heap[0].memory_base,
4179 memory_details.block_count*memory_details.block_size_pinned);
4183 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4184 imemory_data *current_block = memory_details.initial_memory;
4185 for (int i = 0; i < (memory_details.block_count*(total_generation_count - ephemeral_generation_count)); i++, current_block++)
4187 size_t block_size = memory_details.block_size (i);
4188 if (current_block->memory_base != NULL)
4190 virtual_free (current_block->memory_base, block_size);
4195 delete [] memory_details.initial_memory;
4196 memory_details.initial_memory = NULL;
4197 memory_details.initial_normal_heap = NULL;
4198 memory_details.initial_large_heap = NULL;
4199 memory_details.initial_pinned_heap = NULL;
4203 heap_segment* make_initial_segment (int gen, int h_number)
4205 void* mem = memory_details.get_initial_memory (gen, h_number);
4206 size_t size = memory_details.get_initial_size (gen);
4207 gc_oh_num oh = gen_to_oh (gen);
4208 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size, oh, h_number);
4213 void* virtual_alloc (size_t size)
4215 return virtual_alloc(size, false);
4218 void* virtual_alloc (size_t size, bool use_large_pages_p)
4220 size_t requested_size = size;
4222 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4224 gc_heap::reserved_memory_limit =
4225 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4226 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4232 uint32_t flags = VirtualReserveFlags::None;
4233 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4234 if (virtual_alloc_hardware_write_watch)
4236 flags = VirtualReserveFlags::WriteWatch;
4238 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4240 void* prgmem = use_large_pages_p ?
4241 GCToOSInterface::VirtualReserveAndCommitLargePages(requested_size) :
4242 GCToOSInterface::VirtualReserve(requested_size, card_size * card_word_width, flags);
4243 void *aligned_mem = prgmem;
4245 // We don't want (prgmem + size) to be right at the end of the address space
4246 // because we'd have to worry about that everytime we do (address + size).
4247 // We also want to make sure that we leave loh_size_threshold at the end
4248 // so we allocate a small object we don't need to worry about overflow there
4249 // when we do alloc_ptr+size.
4252 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4254 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4256 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4257 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4258 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4266 gc_heap::reserved_memory += requested_size;
4269 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4270 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4275 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4277 size_t seg_size, initial_seg_size;
4281 initial_seg_size = INITIAL_ALLOC;
4282 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4286 initial_seg_size = LHEAP_ALLOC;
4287 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4290 #ifdef MULTIPLE_HEAPS
4293 #endif // HOST_64BIT
4295 if (g_num_processors > 4)
4296 initial_seg_size /= 2;
4297 if (g_num_processors > 8)
4298 initial_seg_size /= 2;
4300 #endif //MULTIPLE_HEAPS
4302 // if seg_size is small but not 0 (0 is default if config not set)
4303 // then set the segment to the minimum size
4304 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4306 // if requested size is between 1 byte and 4MB, use min
4307 if ((seg_size >> 1) && !(seg_size >> 22))
4308 seg_size = 1024*1024*4;
4310 seg_size = initial_seg_size;
4314 seg_size = round_up_power2 (seg_size);
4316 seg_size = round_down_power2 (seg_size);
4317 #endif // HOST_64BIT
4323 gc_heap::compute_new_ephemeral_size()
4325 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4326 size_t padding_size = 0;
4328 for (int i = 0; i <= eph_gen_max; i++)
4330 dynamic_data* dd = dynamic_data_of (i);
4331 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4332 #ifdef RESPECT_LARGE_ALIGNMENT
4333 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4334 #endif //RESPECT_LARGE_ALIGNMENT
4335 #ifdef FEATURE_STRUCTALIGN
4336 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4337 #endif //FEATURE_STRUCTALIGN
4340 padding_size += dd_padding_size (dd);
4341 #endif //SHORT_PLUGS
4344 total_ephemeral_size += eph_gen_starts_size;
4346 #ifdef RESPECT_LARGE_ALIGNMENT
4347 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4348 generation_plan_allocation_start (generation_of (max_generation-1));
4349 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4350 #endif //RESPECT_LARGE_ALIGNMENT
4353 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4354 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4355 #endif //SHORT_PLUGS
4357 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4358 total_ephemeral_size,
4359 padding_size, (total_ephemeral_size - padding_size)));
4363 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4367 gc_heap::soh_get_segment_to_expand()
4369 size_t size = soh_segment_size;
4371 ordered_plug_indices_init = FALSE;
4372 use_bestfit = FALSE;
4374 //compute the size of the new ephemeral heap segment.
4375 compute_new_ephemeral_size();
4377 if ((settings.pause_mode != pause_low_latency) &&
4378 (settings.pause_mode != pause_no_gc)
4379 #ifdef BACKGROUND_GC
4380 && (!gc_heap::background_running_p())
4381 #endif //BACKGROUND_GC
4384 assert (settings.condemned_generation <= max_generation);
4385 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? nullptr :
4386 generation_allocator (generation_of (max_generation)));
4387 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4389 // try to find one in the gen 2 segment list, search backwards because the first segments
4390 // tend to be more compact than the later ones.
4391 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4393 PREFIX_ASSUME(fseg != NULL);
4395 #ifdef SEG_REUSE_STATS
4397 #endif //SEG_REUSE_STATS
4399 heap_segment* seg = ephemeral_heap_segment;
4400 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4402 #ifdef SEG_REUSE_STATS
4404 #endif //SEG_REUSE_STATS
4406 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4408 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4409 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4410 if (settings.condemned_generation == max_generation)
4414 build_ordered_free_spaces (seg);
4415 dprintf (GTC_LOG, ("can use best fit"));
4418 #ifdef SEG_REUSE_STATS
4419 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4420 settings.condemned_generation, try_reuse));
4421 #endif //SEG_REUSE_STATS
4422 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4427 #ifdef SEG_REUSE_STATS
4428 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4429 settings.condemned_generation, try_reuse));
4430 #endif //SEG_REUSE_STATS
4431 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4433 // If we return 0 here, the allocator will think since we are short on end
4434 // of seg we need to trigger a full compacting GC. So if sustained low latency
4435 // is set we should acquire a new seg instead, that way we wouldn't be short.
4436 // The real solution, of course, is to actually implement seg reuse in gen1.
4437 if (settings.pause_mode != pause_sustained_low_latency)
4439 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4440 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4448 heap_segment* result = get_segment (size, gc_oh_num::soh);
4452 #ifdef BACKGROUND_GC
4453 if (current_c_gc_state == c_gc_state_planning)
4455 // When we expand heap during bgc sweep, we set the seg to be swept so
4456 // we'll always look at cards for objects on the new segment.
4457 result->flags |= heap_segment_flags_swept;
4459 #endif //BACKGROUND_GC
4461 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4462 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4463 gc_etw_segment_small_object_heap);
4466 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4470 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4474 #ifdef MULTIPLE_HEAPS
4475 heap_segment_heap (result) = this;
4476 #endif //MULTIPLE_HEAPS
4479 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4484 #pragma warning(default:4706)
4487 //returns 0 in case of allocation failure
4489 gc_heap::get_segment (size_t size, gc_oh_num oh)
4491 assert(oh != gc_oh_num::none);
4492 BOOL uoh_p = (oh == gc_oh_num::loh) || (oh == gc_oh_num::poh);
4493 if (heap_hard_limit)
4496 heap_segment* result = 0;
4498 if (segment_standby_list != 0)
4500 result = segment_standby_list;
4501 heap_segment* last = 0;
4504 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4505 if ((hs >= size) && ((hs / 2) < size))
4507 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4510 heap_segment_next (last) = heap_segment_next (result);
4514 segment_standby_list = heap_segment_next (result);
4521 result = heap_segment_next (result);
4528 init_heap_segment (result);
4529 #ifdef BACKGROUND_GC
4530 if (should_commit_mark_array())
4532 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4533 if (!commit_mark_array_new_seg (__this, result))
4535 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4536 // If we can't use it we need to thread it back.
4537 if (segment_standby_list != 0)
4539 heap_segment_next (result) = segment_standby_list;
4540 segment_standby_list = result;
4544 segment_standby_list = result;
4550 #endif //BACKGROUND_GC
4553 seg_mapping_table_add_segment (result, __this);
4558 void* mem = virtual_alloc (size);
4561 fgm_result.set_fgm (fgm_reserve_segment, size, uoh_p);
4565 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, oh, heap_number);
4571 if (mem < g_gc_lowest_address)
4573 start = (uint8_t*)mem;
4577 start = (uint8_t*)g_gc_lowest_address;
4580 if (((uint8_t*)mem + size) > g_gc_highest_address)
4582 end = (uint8_t*)mem + size;
4586 end = (uint8_t*)g_gc_highest_address;
4589 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, uoh_p) != 0)
4591 virtual_free (mem, size);
4597 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, uoh_p);
4598 virtual_free (mem, size);
4603 seg_mapping_table_add_segment (result, __this);
4607 #ifdef BACKGROUND_GC
4610 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4611 settings.gc_index, current_bgc_state,
4613 bgc_verify_mark_array_cleared (result);
4615 #endif //BACKGROUND_GC
4617 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4621 void gc_heap::release_segment (heap_segment* sg)
4623 ptrdiff_t delta = 0;
4624 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4625 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg, sg);
4628 heap_segment* gc_heap::get_segment_for_uoh (int gen_number, size_t size
4629 #ifdef MULTIPLE_HEAPS
4631 #endif //MULTIPLE_HEAPS
4634 #ifndef MULTIPLE_HEAPS
4636 #endif //MULTIPLE_HEAPS
4637 gc_oh_num oh = gen_to_oh (gen_number);
4638 heap_segment* res = hp->get_segment (size, oh);
4641 #ifdef MULTIPLE_HEAPS
4642 heap_segment_heap (res) = hp;
4643 #endif //MULTIPLE_HEAPS
4644 res->flags |= (gen_number == poh_generation) ?
4645 heap_segment_flags_poh :
4646 heap_segment_flags_loh;
4648 FIRE_EVENT(GCCreateSegment_V1,
4649 heap_segment_mem(res),
4650 (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)),
4651 (gen_number == poh_generation) ?
4652 gc_etw_segment_pinned_object_heap :
4653 gc_etw_segment_large_object_heap);
4655 GCToEEInterface::DiagUpdateGenerationBounds();
4657 #ifdef MULTIPLE_HEAPS
4658 hp->thread_uoh_segment (gen_number, res);
4660 thread_uoh_segment (gen_number, res);
4661 #endif //MULTIPLE_HEAPS
4667 void gc_heap::thread_uoh_segment (int gen_number, heap_segment* new_seg)
4669 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
4671 while (heap_segment_next_rw (seg))
4672 seg = heap_segment_next_rw (seg);
4674 heap_segment_next (seg) = new_seg;
4678 gc_heap::get_uoh_segment (int gen_number, size_t size, BOOL* did_full_compact_gc)
4680 *did_full_compact_gc = FALSE;
4681 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4683 //access to get_segment needs to be serialized
4684 add_saved_spinlock_info (true, me_release, mt_get_large_seg);
4685 leave_spin_lock (&more_space_lock_uoh);
4686 enter_spin_lock (&gc_heap::gc_lock);
4687 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4688 // if a GC happened between here and before we ask for a segment in
4689 // get_uoh_segment, we need to count that GC.
4690 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4692 if (current_full_compact_gc_count > last_full_compact_gc_count)
4694 *did_full_compact_gc = TRUE;
4697 heap_segment* res = get_segment_for_uoh (gen_number, size
4698 #ifdef MULTIPLE_HEAPS
4700 #endif //MULTIPLE_HEAPS
4703 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4704 leave_spin_lock (&gc_heap::gc_lock);
4705 enter_spin_lock (&more_space_lock_uoh);
4706 add_saved_spinlock_info (true, me_acquire, mt_get_large_seg);
4712 #ifdef MULTIPLE_HEAPS
4715 #pragma warning(disable:4035)
4716 static ptrdiff_t get_cycle_count()
4720 #pragma warning(default:4035)
4721 #elif defined(__GNUC__)
4722 static ptrdiff_t get_cycle_count()
4726 __asm__ __volatile__
4727 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4731 #error Unknown compiler
4733 #elif defined(TARGET_AMD64)
4735 extern "C" uint64_t __rdtsc();
4736 #pragma intrinsic(__rdtsc)
4737 static ptrdiff_t get_cycle_count()
4739 return (ptrdiff_t)__rdtsc();
4741 #elif defined(__GNUC__)
4742 static ptrdiff_t get_cycle_count()
4746 __asm__ __volatile__
4747 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4748 return (cyclesHi << 32) | cycles;
4751 extern "C" ptrdiff_t get_cycle_count(void);
4754 static ptrdiff_t get_cycle_count()
4756 // @ARMTODO, @ARM64TODO, @WASMTODO: cycle counter is not exposed to user mode. For now (until we can show this
4757 // makes a difference on the configurations on which we'll run) just return 0. This will result in
4758 // all buffer access times being reported as equal in access_time().
4763 // We may not be on contiguous numa nodes so need to store
4764 // the node index as well.
4765 struct node_heap_count
4775 static uint8_t* sniff_buffer;
4776 static unsigned n_sniff_buffers;
4777 static unsigned cur_sniff_index;
4779 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
4780 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
4781 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
4782 static uint16_t proc_no_to_numa_node[MAX_SUPPORTED_CPUS];
4783 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
4784 // Note this is the total numa nodes GC heaps are on. There might be
4785 // more on the machine if GC threads aren't using all of them.
4786 static uint16_t total_numa_nodes;
4787 static node_heap_count heaps_on_node[MAX_SUPPORTED_NODES];
4789 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
4791 ptrdiff_t start_cycles = get_cycle_count();
4792 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
4793 assert (sniff == 0);
4794 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
4795 // add sniff here just to defeat the optimizer
4796 elapsed_cycles += sniff;
4797 return (int) elapsed_cycles;
4801 static BOOL init(int n_heaps)
4803 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
4804 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
4806 n_sniff_buffers = n_heaps*2+1;
4807 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
4808 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
4809 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
4814 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
4815 if (sniff_buffer == 0)
4817 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
4820 //can not enable gc numa aware, force all heaps to be in
4821 //one numa node by filling the array with all 0s
4822 if (!GCToOSInterface::CanEnableGCNumaAware())
4823 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
4828 static void init_cpu_mapping(int heap_number)
4830 if (GCToOSInterface::CanGetCurrentProcessorNumber())
4832 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber();
4833 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
4837 static void mark_heap(int heap_number)
4839 if (GCToOSInterface::CanGetCurrentProcessorNumber())
4842 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
4843 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
4846 static int select_heap(alloc_context* acontext)
4849 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
4852 if (GCToOSInterface::CanGetCurrentProcessorNumber())
4854 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber();
4855 return proc_no_to_heap_no[proc_no];
4858 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
4859 sniff_index %= n_sniff_buffers;
4862 int best_access_time = 1000*1000*1000;
4863 int second_best_access_time = best_access_time;
4865 uint8_t *l_sniff_buffer = sniff_buffer;
4866 unsigned l_n_sniff_buffers = n_sniff_buffers;
4867 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
4869 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
4870 if (this_access_time < best_access_time)
4872 second_best_access_time = best_access_time;
4873 best_access_time = this_access_time;
4874 best_heap = heap_number;
4876 else if (this_access_time < second_best_access_time)
4878 second_best_access_time = this_access_time;
4882 if (best_access_time*2 < second_best_access_time)
4884 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
4886 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
4890 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
4896 static bool can_find_heap_fast()
4898 return GCToOSInterface::CanGetCurrentProcessorNumber();
4901 static uint16_t find_heap_no_from_proc_no(uint16_t proc_no)
4903 return proc_no_to_heap_no[proc_no];
4906 static uint16_t find_proc_no_from_heap_no(int heap_number)
4908 return heap_no_to_proc_no[heap_number];
4911 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
4913 heap_no_to_proc_no[heap_number] = proc_no;
4916 static uint16_t find_numa_node_from_heap_no(int heap_number)
4918 return heap_no_to_numa_node[heap_number];
4921 static uint16_t find_numa_node_from_proc_no (uint16_t proc_no)
4923 return proc_no_to_numa_node[proc_no];
4926 static void set_numa_node_for_heap_and_proc(int heap_number, uint16_t proc_no, uint16_t numa_node)
4928 heap_no_to_numa_node[heap_number] = numa_node;
4929 proc_no_to_numa_node[proc_no] = numa_node;
4932 static void init_numa_node_to_heap_map(int nheaps)
4934 // Called right after GCHeap::Init() for each heap
4935 // For each NUMA node used by the heaps, the
4936 // numa_node_to_heap_map[numa_node] is set to the first heap number on that node and
4937 // numa_node_to_heap_map[numa_node + 1] is set to the first heap number not on that node
4938 // Set the start of the heap number range for the first NUMA node
4939 numa_node_to_heap_map[heap_no_to_numa_node[0]] = 0;
4940 total_numa_nodes = 0;
4941 memset (heaps_on_node, 0, sizeof (heaps_on_node));
4942 heaps_on_node[0].node_no = heap_no_to_numa_node[0];
4943 heaps_on_node[0].heap_count = 1;
4945 for (int i=1; i < nheaps; i++)
4947 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
4950 heaps_on_node[total_numa_nodes].node_no = heap_no_to_numa_node[i];
4952 // Set the end of the heap number range for the previous NUMA node
4953 numa_node_to_heap_map[heap_no_to_numa_node[i-1] + 1] =
4954 // Set the start of the heap number range for the current NUMA node
4955 numa_node_to_heap_map[heap_no_to_numa_node[i]] = (uint16_t)i;
4957 (heaps_on_node[total_numa_nodes].heap_count)++;
4960 // Set the end of the heap range for the last NUMA node
4961 numa_node_to_heap_map[heap_no_to_numa_node[nheaps-1] + 1] = (uint16_t)nheaps; //mark the end with nheaps
4965 // TODO: curently this doesn't work with GCHeapAffinitizeMask/GCHeapAffinitizeRanges
4966 // because the heaps may not be on contiguous active procs.
4968 // This is for scenarios where GCHeapCount is specified as something like
4969 // (g_num_active_processors - 2) to allow less randomization to the Server GC threads.
4970 // In this case we want to assign the right heaps to those procs, ie if they share
4971 // the same numa node we want to assign local heaps to those procs. Otherwise we
4972 // let the heap balancing mechanism take over for now.
4973 static void distribute_other_procs()
4975 if (affinity_config_specified_p)
4978 uint16_t proc_no = 0;
4979 uint16_t node_no = 0;
4981 int start_heap = -1;
4983 int current_node_no = -1;
4984 int current_heap_on_node = -1;
4986 for (int i = gc_heap::n_heaps; i < (int)g_num_active_processors; i++)
4988 if (!GCToOSInterface::GetProcessorForHeap (i, &proc_no, &node_no))
4991 int start_heap = (int)numa_node_to_heap_map[node_no];
4992 int end_heap = (int)(numa_node_to_heap_map[node_no + 1]);
4994 if ((end_heap - start_heap) > 0)
4996 if (node_no == current_node_no)
4998 // We already iterated through all heaps on this node, don't add more procs to these
5000 if (current_heap_on_node >= end_heap)
5007 current_node_no = node_no;
5008 current_heap_on_node = start_heap;
5011 proc_no_to_heap_no[proc_no] = current_heap_on_node;
5012 proc_no_to_numa_node[proc_no] = node_no;
5014 current_heap_on_node++;
5019 static void get_heap_range_for_heap(int hn, int* start, int* end)
5021 uint16_t numa_node = heap_no_to_numa_node[hn];
5022 *start = (int)numa_node_to_heap_map[numa_node];
5023 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5026 // This gets the next valid numa node index starting at current_index+1.
5027 // It assumes that current_index is a valid node index.
5028 // If current_index+1 is at the end this will start at the beginning. So this will
5029 // always return a valid node index, along with that node's start/end heaps.
5030 static uint16_t get_next_numa_node (uint16_t current_index, int* start, int* end)
5032 int start_index = current_index + 1;
5033 int nheaps = gc_heap::n_heaps;
5035 bool found_node_with_heaps_p = false;
5038 int start_heap = (int)numa_node_to_heap_map[start_index];
5039 int end_heap = (int)numa_node_to_heap_map[start_index + 1];
5040 if (start_heap == nheaps)
5042 // This is the last node.
5047 if ((end_heap - start_heap) == 0)
5049 // This node has no heaps.
5054 found_node_with_heaps_p = true;
5055 *start = start_heap;
5058 } while (!found_node_with_heaps_p);
5063 uint8_t* heap_select::sniff_buffer;
5064 unsigned heap_select::n_sniff_buffers;
5065 unsigned heap_select::cur_sniff_index;
5066 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5067 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5068 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5069 uint16_t heap_select::proc_no_to_numa_node[MAX_SUPPORTED_CPUS];
5070 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5071 uint16_t heap_select::total_numa_nodes;
5072 node_heap_count heap_select::heaps_on_node[MAX_SUPPORTED_NODES];
5074 #ifdef HEAP_BALANCE_INSTRUMENTATION
5075 // This records info we use to look at effect of different strategies
5076 // for heap balancing.
5077 struct heap_balance_info
5080 // This also encodes when we detect the thread runs on
5081 // different proc during a balance attempt. Sometimes
5082 // I observe this happens multiple times during one attempt!
5083 // If this happens, I just record the last proc we observe
5086 // This records the final alloc_heap for the thread.
5088 // This also encodes the reason why we needed to set_home_heap
5089 // in balance_heaps.
5090 // If we set it because the home heap is not the same as the proc,
5093 // If we set ideal proc, we set the 2nd MSB.
5098 // This means inbetween each GC we can log at most this many entries per proc.
5099 // This is usually enough. Most of the time we only need to log something every 128k
5100 // of allocations in balance_heaps and gen0 budget is <= 200mb.
5101 #define default_max_hb_heap_balance_info 4096
5103 struct heap_balance_info_proc
5107 heap_balance_info hb_info[default_max_hb_heap_balance_info];
5110 struct heap_balance_info_numa
5112 heap_balance_info_proc* hb_info_procs;
5115 uint64_t start_raw_ts = 0;
5116 bool cpu_group_enabled_p = false;
5117 uint32_t procs_per_numa_node = 0;
5118 uint16_t total_numa_nodes_on_machine = 0;
5119 uint32_t procs_per_cpu_group = 0;
5120 uint16_t total_cpu_groups_on_machine = 0;
5121 // Note this is still on one of the numa nodes, so we'll incur a remote access
5123 heap_balance_info_numa* hb_info_numa_nodes = NULL;
5125 // TODO: This doesn't work for multiple nodes per CPU group yet.
5126 int get_proc_index_numa (int proc_no, int* numa_no)
5128 if (total_numa_nodes_on_machine == 1)
5135 if (cpu_group_enabled_p)
5137 // see vm\gcenv.os.cpp GroupProcNo implementation.
5138 *numa_no = proc_no >> 6;
5139 return (proc_no % 64);
5143 *numa_no = proc_no / procs_per_numa_node;
5144 return (proc_no % procs_per_numa_node);
5149 // We could consider optimizing it so we don't need to get the tid
5150 // everytime but it's not very expensive to get.
5151 void add_to_hb_numa (
5155 bool multiple_procs_p,
5159 int tid = (int)GCToOSInterface::GetCurrentThreadIdForLogging ();
5160 uint64_t timestamp = RawGetHighPrecisionTimeStamp ();
5162 int saved_proc_no = proc_no;
5164 proc_no = get_proc_index_numa (proc_no, &numa_no);
5166 heap_balance_info_numa* hb_info_numa_node = &hb_info_numa_nodes[numa_no];
5168 heap_balance_info_proc* hb_info_proc = &(hb_info_numa_node->hb_info_procs[proc_no]);
5169 int index = hb_info_proc->index;
5170 int count = hb_info_proc->count;
5174 // Too much info inbetween GCs. This can happen if the thread is scheduled on a different
5175 // processor very often so it caused us to log many entries due to that reason. You could
5176 // increase default_max_hb_heap_balance_info but this usually indicates a problem that
5177 // should be investigated.
5178 dprintf (HEAP_BALANCE_LOG, ("too much info between GCs, already logged %d entries", index));
5179 GCToOSInterface::DebugBreak ();
5181 heap_balance_info* hb_info = &(hb_info_proc->hb_info[index]);
5183 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMP[p%3d->%3d(i:%3d), N%d] #%4d: %I64d, tid %d, ah: %d, m: %d, p: %d, i: %d",
5184 saved_proc_no, proc_no, ideal_proc_no, numa_no, index,
5185 (timestamp - start_raw_ts) / 1000, tid, alloc_heap, (int)multiple_procs_p, (int)(!alloc_count_p), (int)set_ideal_p));
5187 if (multiple_procs_p)
5189 tid |= (1 << (sizeof (tid) * 8 - 1));
5194 alloc_heap |= (1 << (sizeof (alloc_heap) * 8 - 1));
5199 alloc_heap |= (1 << (sizeof (alloc_heap) * 8 - 2));
5202 hb_info->timestamp = timestamp;
5204 hb_info->alloc_heap = alloc_heap;
5205 hb_info->ideal_proc_no = ideal_proc_no;
5206 (hb_info_proc->index)++;
5209 const int hb_log_buffer_size = 1024;
5210 static char hb_log_buffer[hb_log_buffer_size];
5211 int last_hb_recorded_gc_index = -1;
5212 #endif //HEAP_BALANCE_INSTRUMENTATION
5214 // This logs what we recorded in balance_heaps
5215 // The format for this is
5217 // [ms since last GC end]
5219 // all elements we stored before this GC for this CPU in the format
5220 // timestamp,tid, alloc_heap_no
5221 // repeat this for each CPU
5223 // the timestamp here is just the result of calling QPC,
5224 // it's not converted to ms. The conversion will be done when we process
5226 void gc_heap::hb_log_balance_activities()
5228 #ifdef HEAP_BALANCE_INSTRUMENTATION
5229 char* log_buffer = hb_log_buffer;
5231 uint64_t now = GetHighPrecisionTimeStamp();
5232 size_t time_since_last_gc_ms = (size_t)((now - last_gc_end_time_us) / 1000);
5233 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMP%Id - %Id = %Id", now, last_gc_end_time_ms, time_since_last_gc_ms));
5235 // We want to get the min and the max timestamp for all procs because it helps with our post processing
5236 // to know how big an array to allocate to display the history inbetween the GCs.
5237 uint64_t min_timestamp = 0xffffffffffffffff;
5238 uint64_t max_timestamp = 0;
5240 for (int numa_node_index = 0; numa_node_index < total_numa_nodes_on_machine; numa_node_index++)
5242 heap_balance_info_proc* hb_info_procs = hb_info_numa_nodes[numa_node_index].hb_info_procs;
5243 for (int proc_index = 0; proc_index < (int)procs_per_numa_node; proc_index++)
5245 heap_balance_info_proc* hb_info_proc = &hb_info_procs[proc_index];
5246 int total_entries_on_proc = hb_info_proc->index;
5248 if (total_entries_on_proc > 0)
5250 min_timestamp = min (min_timestamp, hb_info_proc->hb_info[0].timestamp);
5251 max_timestamp = max (max_timestamp, hb_info_proc->hb_info[total_entries_on_proc - 1].timestamp);
5256 dprintf (HEAP_BALANCE_LOG, ("[GCA#%Id %Id-%I64d-%I64d]",
5257 settings.gc_index, time_since_last_gc_ms, (min_timestamp - start_raw_ts), (max_timestamp - start_raw_ts)));
5259 if (last_hb_recorded_gc_index == (int)settings.gc_index)
5261 GCToOSInterface::DebugBreak ();
5264 last_hb_recorded_gc_index = (int)settings.gc_index;
5266 // When we print out the proc index we need to convert it to the actual proc index (this is contiguous).
5267 // It helps with post processing.
5268 for (int numa_node_index = 0; numa_node_index < total_numa_nodes_on_machine; numa_node_index++)
5270 heap_balance_info_proc* hb_info_procs = hb_info_numa_nodes[numa_node_index].hb_info_procs;
5271 for (int proc_index = 0; proc_index < (int)procs_per_numa_node; proc_index++)
5273 heap_balance_info_proc* hb_info_proc = &hb_info_procs[proc_index];
5274 int total_entries_on_proc = hb_info_proc->index;
5275 if (total_entries_on_proc > 0)
5277 int total_exec_time_ms =
5278 (int)((double)(hb_info_proc->hb_info[total_entries_on_proc - 1].timestamp - hb_info_proc->hb_info[0].timestamp) * qpf_ms);
5279 dprintf (HEAP_BALANCE_LOG, ("[p%d]-%d-%dms",
5280 (proc_index + numa_node_index * procs_per_numa_node), total_entries_on_proc, total_exec_time_ms));
5283 for (int i = 0; i < hb_info_proc->index; i++)
5285 heap_balance_info* hb_info = &hb_info_proc->hb_info[i];
5286 bool multiple_procs_p = false;
5287 bool alloc_count_p = true;
5288 bool set_ideal_p = false;
5289 int tid = hb_info->tid;
5290 int alloc_heap = hb_info->alloc_heap;
5292 if (tid & (1 << (sizeof (tid) * 8 - 1)))
5294 multiple_procs_p = true;
5295 tid &= ~(1 << (sizeof (tid) * 8 - 1));
5298 if (alloc_heap & (1 << (sizeof (alloc_heap) * 8 - 1)))
5300 alloc_count_p = false;
5301 alloc_heap &= ~(1 << (sizeof (alloc_heap) * 8 - 1));
5304 if (alloc_heap & (1 << (sizeof (alloc_heap) * 8 - 2)))
5307 alloc_heap &= ~(1 << (sizeof (alloc_heap) * 8 - 2));
5310 // TODO - This assumes ideal proc is in the same cpu group which is not true
5311 // when we don't have CPU groups.
5312 int ideal_proc_no = hb_info->ideal_proc_no;
5313 int ideal_node_no = -1;
5314 ideal_proc_no = get_proc_index_numa (ideal_proc_no, &ideal_node_no);
5315 ideal_proc_no = ideal_proc_no + ideal_node_no * procs_per_numa_node;
5317 dprintf (HEAP_BALANCE_LOG, ("%I64d,%d,%d,%d%s%s%s",
5318 (hb_info->timestamp - start_raw_ts),
5322 (multiple_procs_p ? "|m" : ""), (!alloc_count_p ? "|p" : ""), (set_ideal_p ? "|i" : "")));
5327 for (int numa_node_index = 0; numa_node_index < total_numa_nodes_on_machine; numa_node_index++)
5329 heap_balance_info_proc* hb_info_procs = hb_info_numa_nodes[numa_node_index].hb_info_procs;
5330 for (int proc_index = 0; proc_index < (int)procs_per_numa_node; proc_index++)
5332 heap_balance_info_proc* hb_info_proc = &hb_info_procs[proc_index];
5333 hb_info_proc->index = 0;
5336 #endif //HEAP_BALANCE_INSTRUMENTATION
5339 // The format for this is
5342 // h0_new_alloc, h1_new_alloc, ...
5344 void gc_heap::hb_log_new_allocation()
5346 #ifdef HEAP_BALANCE_INSTRUMENTATION
5347 char* log_buffer = hb_log_buffer;
5349 int desired_alloc_mb = (int)(dd_desired_allocation (g_heaps[0]->dynamic_data_of (0)) / 1024 / 1024);
5351 int buffer_pos = sprintf_s (hb_log_buffer, hb_log_buffer_size, "[GC_alloc_mb]\n");
5352 for (int numa_node_index = 0; numa_node_index < heap_select::total_numa_nodes; numa_node_index++)
5354 int node_allocated_mb = 0;
5356 // I'm printing out the budget here instead of the numa node index so we know how much
5357 // of the budget we consumed.
5358 buffer_pos += sprintf_s (hb_log_buffer + buffer_pos, hb_log_buffer_size - buffer_pos, "[N#%3d]",
5362 int heaps_on_node = heap_select::heaps_on_node[numa_node_index].heap_count;
5364 for (int heap_index = 0; heap_index < heaps_on_node; heap_index++)
5366 int actual_heap_index = heap_index + numa_node_index * heaps_on_node;
5367 gc_heap* hp = g_heaps[actual_heap_index];
5368 dynamic_data* dd0 = hp->dynamic_data_of (0);
5369 int allocated_mb = (int)((dd_desired_allocation (dd0) - dd_new_allocation (dd0)) / 1024 / 1024);
5370 node_allocated_mb += allocated_mb;
5371 buffer_pos += sprintf_s (hb_log_buffer + buffer_pos, hb_log_buffer_size - buffer_pos, "%d,",
5375 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPN#%d a %dmb(%dmb)", numa_node_index, node_allocated_mb, desired_alloc_mb));
5377 buffer_pos += sprintf_s (hb_log_buffer + buffer_pos, hb_log_buffer_size - buffer_pos, "\n");
5380 dprintf (HEAP_BALANCE_LOG, ("%s", hb_log_buffer));
5381 #endif //HEAP_BALANCE_INSTRUMENTATION
5384 BOOL gc_heap::create_thread_support (int number_of_heaps)
5387 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5391 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5395 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5406 destroy_thread_support();
5412 void gc_heap::destroy_thread_support ()
5414 if (ee_suspend_event.IsValid())
5416 ee_suspend_event.CloseEvent();
5418 if (gc_start_event.IsValid())
5420 gc_start_event.CloseEvent();
5424 bool get_proc_and_numa_for_heap (int heap_number)
5429 bool res = GCToOSInterface::GetProcessorForHeap (heap_number, &proc_no, &node_no);
5432 heap_select::set_proc_no_for_heap (heap_number, proc_no);
5433 if (node_no != NUMA_NODE_UNDEFINED)
5435 heap_select::set_numa_node_for_heap_and_proc (heap_number, proc_no, node_no);
5442 void set_thread_affinity_for_heap (int heap_number, uint16_t proc_no)
5444 if (!GCToOSInterface::SetThreadAffinity (proc_no))
5446 dprintf (1, ("Failed to set thread affinity for GC thread %d on proc #%d", heap_number, proc_no));
5450 bool gc_heap::create_gc_thread ()
5452 dprintf (3, ("Creating gc thread\n"));
5453 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5457 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5459 void gc_heap::gc_thread_function ()
5461 assert (gc_done_event.IsValid());
5462 assert (gc_start_event.IsValid());
5463 dprintf (3, ("gc thread started"));
5465 heap_select::init_cpu_mapping(heap_number);
5469 assert (!gc_t_join.joined());
5471 if (heap_number == 0)
5473 uint32_t wait_result = gc_heap::ee_suspend_event.Wait(gradual_decommit_in_progress_p ? DECOMMIT_TIME_STEP_MILLISECONDS : INFINITE, FALSE);
5474 if (wait_result == WAIT_TIMEOUT)
5476 gradual_decommit_in_progress_p = decommit_step ();
5480 suspended_start_time = GetHighPrecisionTimeStamp();
5481 BEGIN_TIMING(suspend_ee_during_log);
5482 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5483 END_TIMING(suspend_ee_during_log);
5485 proceed_with_gc_p = TRUE;
5487 if (!should_proceed_with_gc())
5489 update_collection_counts_for_no_gc();
5490 proceed_with_gc_p = FALSE;
5494 settings.init_mechanisms();
5495 gc_start_event.Set();
5497 dprintf (3, ("%d gc thread waiting...", heap_number));
5501 gc_start_event.Wait(INFINITE, FALSE);
5502 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5505 assert ((heap_number == 0) || proceed_with_gc_p);
5507 if (proceed_with_gc_p)
5509 garbage_collect (GCHeap::GcCondemnedGeneration);
5511 if (pm_trigger_full_gc)
5513 garbage_collect_pm_full_gc();
5517 if (heap_number == 0)
5519 if (proceed_with_gc_p && (!settings.concurrent))
5524 #ifdef BACKGROUND_GC
5525 recover_bgc_settings();
5526 #endif //BACKGROUND_GC
5528 #ifdef MULTIPLE_HEAPS
5529 for (int i = 0; i < gc_heap::n_heaps; i++)
5531 gc_heap* hp = gc_heap::g_heaps[i];
5532 hp->add_saved_spinlock_info (false, me_release, mt_block_gc);
5533 leave_spin_lock(&hp->more_space_lock_soh);
5535 #endif //MULTIPLE_HEAPS
5537 gc_heap::gc_started = FALSE;
5539 #ifdef BACKGROUND_GC
5540 gc_heap::add_bgc_pause_duration_0();
5541 #endif //BACKGROUND_GC
5542 BEGIN_TIMING(restart_ee_during_log);
5543 GCToEEInterface::RestartEE(TRUE);
5544 END_TIMING(restart_ee_during_log);
5545 process_sync_log_stats();
5547 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5548 leave_spin_lock (&gc_heap::gc_lock);
5550 gc_heap::internal_gc_done = true;
5552 if (proceed_with_gc_p)
5556 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5557 // we still need to set the gc_done_event for those threads.
5558 for (int i = 0; i < gc_heap::n_heaps; i++)
5560 gc_heap* hp = gc_heap::g_heaps[i];
5565 // check if we should do some decommitting
5566 if (gradual_decommit_in_progress_p)
5568 gradual_decommit_in_progress_p = decommit_step ();
5573 int spin_count = 32 * (gc_heap::n_heaps - 1);
5575 // wait until RestartEE has progressed to a stage where we can restart user threads
5576 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5578 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5585 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5588 #endif //MULTIPLE_HEAPS
5590 bool gc_heap::virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number)
5592 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5593 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5594 // a host. This will need to be added later.
5595 #if !defined(FEATURE_CORECLR) && !defined(BUILD_AS_STANDALONE)
5596 if (!CLRMemoryHosted())
5599 if (GCToOSInterface::CanEnableGCNumaAware())
5601 uint16_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5602 if (GCToOSInterface::VirtualCommit (addr, size, numa_node))
5606 #else //MULTIPLE_HEAPS && !FEATURE_REDHAWK
5607 UNREFERENCED_PARAMETER(h_number);
5608 #endif //MULTIPLE_HEAPS && !FEATURE_REDHAWK
5610 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5611 return GCToOSInterface::VirtualCommit(addr, size);
5614 bool gc_heap::virtual_commit (void* address, size_t size, gc_oh_num oh, int h_number, bool* hard_limit_exceeded_p)
5617 assert (heap_hard_limit == 0);
5618 #endif //!HOST_64BIT
5620 if (heap_hard_limit)
5622 check_commit_cs.Enter();
5623 bool exceeded_p = false;
5625 if (heap_hard_limit_oh[0] != 0)
5627 if ((oh != gc_oh_num::none) && (committed_by_oh[oh] + size) > heap_hard_limit_oh[oh])
5632 else if ((current_total_committed + size) > heap_hard_limit)
5634 dprintf (1, ("%Id + %Id = %Id > limit %Id ",
5635 current_total_committed, size,
5636 (current_total_committed + size),
5644 committed_by_oh[oh] += size;
5645 current_total_committed += size;
5647 current_total_committed_bookkeeping += size;
5650 check_commit_cs.Leave();
5652 if (hard_limit_exceeded_p)
5653 *hard_limit_exceeded_p = exceeded_p;
5657 dprintf (1, ("can't commit %Ix for %Id bytes > HARD LIMIT %Id", (size_t)address, size, heap_hard_limit));
5662 // If it's a valid heap number it means it's commiting for memory on the GC heap.
5663 // In addition if large pages is enabled, we set commit_succeeded_p to true because memory is already committed.
5664 bool commit_succeeded_p = ((h_number >= 0) ? (use_large_pages_p ? true :
5665 virtual_alloc_commit_for_heap (address, size, h_number)) :
5666 GCToOSInterface::VirtualCommit(address, size));
5668 if (!commit_succeeded_p && heap_hard_limit)
5670 check_commit_cs.Enter();
5671 committed_by_oh[oh] -= size;
5673 dprintf (1, ("commit failed, updating %Id to %Id",
5674 current_total_committed, (current_total_committed - size)));
5675 current_total_committed -= size;
5677 current_total_committed_bookkeeping -= size;
5679 check_commit_cs.Leave();
5681 return commit_succeeded_p;
5684 bool gc_heap::virtual_decommit (void* address, size_t size, gc_oh_num oh, int h_number)
5687 assert (heap_hard_limit == 0);
5688 #endif //!HOST_64BIT
5690 bool decommit_succeeded_p = GCToOSInterface::VirtualDecommit (address, size);
5692 if (decommit_succeeded_p && heap_hard_limit)
5694 check_commit_cs.Enter();
5695 committed_by_oh[oh] -= size;
5696 current_total_committed -= size;
5698 current_total_committed_bookkeeping -= size;
5699 check_commit_cs.Leave();
5702 return decommit_succeeded_p;
5705 void gc_heap::virtual_free (void* add, size_t allocated_size, heap_segment* sg)
5707 assert(!heap_hard_limit);
5708 bool release_succeeded_p = GCToOSInterface::VirtualRelease (add, allocated_size);
5709 if (release_succeeded_p)
5711 reserved_memory -= allocated_size;
5712 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
5713 allocated_size, (size_t)add, (size_t)((uint8_t*)add + allocated_size)));
5723 // If we want to save space we can have a pool of plug_and_gap's instead of
5724 // always having 2 allocated for each pinned plug.
5725 gap_reloc_pair saved_pre_plug;
5726 // If we decide to not compact, we need to restore the original values.
5727 gap_reloc_pair saved_pre_plug_reloc;
5729 gap_reloc_pair saved_post_plug;
5731 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5732 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5734 // We know these cases will be rare so we can optimize this to be only allocated on demand.
5735 gap_reloc_pair saved_post_plug_reloc;
5737 // We need to calculate this after we are done with plan phase and before compact
5738 // phase because compact phase will change the bricks so relocate_address will no
5740 uint8_t* saved_pre_plug_info_reloc_start;
5742 // We need to save this because we will have no way to calculate it, unlike the
5743 // pre plug info start which is right before this plug.
5744 uint8_t* saved_post_plug_info_start;
5747 uint8_t* allocation_context_start_region;
5748 #endif //SHORT_PLUGS
5750 // How the bits in these bytes are organized:
5752 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5753 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5758 // We are seeing this is getting corrupted for a PP with a NP after.
5759 // Save it when we first set it and make sure it doesn't change.
5760 gap_reloc_pair saved_post_plug_debug;
5763 size_t get_max_short_bits()
5765 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5769 size_t get_pre_short_start_bit ()
5771 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5776 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5779 void set_pre_short()
5781 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5784 void set_pre_short_bit (size_t bit)
5786 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5789 BOOL pre_short_bit_p (size_t bit)
5791 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5794 #ifdef COLLECTIBLE_CLASS
5795 void set_pre_short_collectible()
5800 BOOL pre_short_collectible_p()
5802 return (saved_pre_p & 2);
5804 #endif //COLLECTIBLE_CLASS
5807 size_t get_post_short_start_bit ()
5809 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5814 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5817 void set_post_short()
5819 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5822 void set_post_short_bit (size_t bit)
5824 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5827 BOOL post_short_bit_p (size_t bit)
5829 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5832 #ifdef COLLECTIBLE_CLASS
5833 void set_post_short_collectible()
5838 BOOL post_short_collectible_p()
5840 return (saved_post_p & 2);
5842 #endif //COLLECTIBLE_CLASS
5844 uint8_t* get_plug_address() { return first; }
5846 BOOL has_pre_plug_info() { return saved_pre_p; }
5847 BOOL has_post_plug_info() { return saved_post_p; }
5849 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5850 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5851 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5852 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5854 // We need to temporarily recover the shortened plugs for compact phase so we can
5855 // copy over the whole plug and their related info (mark bits/cards). But we will
5856 // need to set the artificial gap back so compact phase can keep reading the plug info.
5857 // We also need to recover the saved info because we'll need to recover it later.
5859 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5860 // it again to recover the artificial gap.
5861 void swap_pre_plug_and_saved()
5863 gap_reloc_pair temp;
5864 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5865 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5866 saved_pre_plug_reloc = temp;
5869 void swap_post_plug_and_saved()
5871 gap_reloc_pair temp;
5872 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5873 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5874 saved_post_plug_reloc = temp;
5877 void swap_pre_plug_and_saved_for_profiler()
5879 gap_reloc_pair temp;
5880 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5881 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5882 saved_pre_plug = temp;
5885 void swap_post_plug_and_saved_for_profiler()
5887 gap_reloc_pair temp;
5888 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5889 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5890 saved_post_plug = temp;
5893 // We should think about whether it's really necessary to have to copy back the pre plug
5894 // info since it was already copied during compacting plugs. But if a plug doesn't move
5895 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5896 void recover_plug_info()
5900 if (gc_heap::settings.compaction)
5902 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5904 &saved_pre_plug_reloc,
5905 saved_pre_plug_info_reloc_start));
5906 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5910 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5913 (first - sizeof (plug_and_gap))));
5914 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5920 if (gc_heap::settings.compaction)
5922 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5924 &saved_post_plug_reloc,
5925 saved_post_plug_info_start));
5926 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5930 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5933 saved_post_plug_info_start));
5934 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5941 void gc_mechanisms::init_mechanisms()
5943 condemned_generation = 0;
5944 promotion = FALSE;//TRUE;
5946 #ifdef FEATURE_LOH_COMPACTION
5947 loh_compaction = gc_heap::loh_compaction_requested();
5949 loh_compaction = FALSE;
5950 #endif //FEATURE_LOH_COMPACTION
5951 heap_expansion = FALSE;
5954 elevation_reduced = FALSE;
5955 found_finalizers = FALSE;
5956 #ifdef BACKGROUND_GC
5957 background_p = gc_heap::background_running_p() != FALSE;
5958 allocations_allowed = TRUE;
5959 #endif //BACKGROUND_GC
5961 entry_memory_load = 0;
5962 entry_available_physical_mem = 0;
5963 exit_memory_load = 0;
5966 stress_induced = FALSE;
5967 #endif // STRESS_HEAP
5970 void gc_mechanisms::first_init()
5973 gen0_reduction_count = 0;
5974 should_lock_elevation = FALSE;
5975 elevation_locked_count = 0;
5976 reason = reason_empty;
5977 #ifdef BACKGROUND_GC
5978 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5980 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5981 if (debug_pause_mode >= 0)
5983 assert (debug_pause_mode <= pause_sustained_low_latency);
5984 pause_mode = (gc_pause_mode)debug_pause_mode;
5987 #else //BACKGROUND_GC
5988 pause_mode = pause_batch;
5989 #endif //BACKGROUND_GC
5994 void gc_mechanisms::record (gc_history_global* history)
5996 #ifdef MULTIPLE_HEAPS
5997 history->num_heaps = gc_heap::n_heaps;
5999 history->num_heaps = 1;
6000 #endif //MULTIPLE_HEAPS
6002 history->condemned_generation = condemned_generation;
6003 history->gen0_reduction_count = gen0_reduction_count;
6004 history->reason = reason;
6005 history->pause_mode = (int)pause_mode;
6006 history->mem_pressure = entry_memory_load;
6007 history->global_mechanisms_p = 0;
6009 // start setting the boolean values.
6011 history->set_mechanism_p (global_concurrent);
6014 history->set_mechanism_p (global_compaction);
6017 history->set_mechanism_p (global_promotion);
6020 history->set_mechanism_p (global_demotion);
6023 history->set_mechanism_p (global_card_bundles);
6025 if (elevation_reduced)
6026 history->set_mechanism_p (global_elevation);
6029 /**********************************
6030 called at the beginning of GC to fix the allocated size to
6031 what is really allocated, or to turn the free area into an unused object
6032 It needs to be called after all of the other allocation contexts have been
6033 fixed since it relies on alloc_allocated.
6034 ********************************/
6036 //for_gc_p indicates that the work is being done for GC,
6037 //as opposed to concurrent heap verification
6038 void gc_heap::fix_youngest_allocation_area()
6040 // The gen 0 alloc context is never used for allocation in the allocator path. It's
6041 // still used in the allocation path during GCs.
6042 assert (generation_allocation_pointer (youngest_generation) == nullptr);
6043 assert (generation_allocation_limit (youngest_generation) == nullptr);
6044 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
6047 void gc_heap::fix_uoh_allocation_area()
6049 for (int i = uoh_start_generation; i < total_generation_count; i++)
6052 alloc_context* acontext =
6054 generation_alloc_context (generation_of (i));
6055 assert (acontext->alloc_ptr == 0);
6056 assert (acontext->alloc_limit == 0);
6059 dprintf (3, ("UOH alloc context: gen: %Ix, ptr: %Ix, limit %Ix",
6060 i, (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
6061 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
6064 acontext->alloc_ptr = 0;
6065 acontext->alloc_limit = acontext->alloc_ptr;
6072 //for_gc_p indicates that the work is being done for GC,
6073 //as opposed to concurrent heap verification
6074 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
6077 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
6079 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
6081 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
6084 uint8_t* point = acontext->alloc_ptr;
6087 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
6088 // the allocation area was from the free list
6089 // it was shortened by Align (min_obj_size) to make room for
6090 // at least the shortest unused object
6091 size += Align (min_obj_size, align_const);
6092 assert ((size >= Align (min_obj_size)));
6094 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
6095 (size_t)point + size ));
6096 make_unused_array (point, size);
6100 generation_free_obj_space (generation_of (0)) += size;
6101 alloc_contexts_used ++;
6107 alloc_allocated = acontext->alloc_ptr;
6108 assert (heap_segment_allocated (ephemeral_heap_segment) <=
6109 heap_segment_committed (ephemeral_heap_segment));
6110 alloc_contexts_used ++;
6115 // We need to update the alloc_bytes to reflect the portion that we have not used
6116 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
6117 total_alloc_bytes_soh -= (acontext->alloc_limit - acontext->alloc_ptr);
6119 acontext->alloc_ptr = 0;
6120 acontext->alloc_limit = acontext->alloc_ptr;
6124 //used by the heap verification for concurrent gc.
6125 //it nulls out the words set by fix_allocation_context for heap_verification
6126 void repair_allocation (gc_alloc_context* acontext, void*)
6128 uint8_t* point = acontext->alloc_ptr;
6132 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6133 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6134 memclr (acontext->alloc_ptr - plug_skew,
6135 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
6139 void void_allocation (gc_alloc_context* acontext, void*)
6141 uint8_t* point = acontext->alloc_ptr;
6145 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6146 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6147 acontext->alloc_ptr = 0;
6148 acontext->alloc_limit = acontext->alloc_ptr;
6152 void gc_heap::repair_allocation_contexts (BOOL repair_p)
6154 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
6157 struct fix_alloc_context_args
6163 void fix_alloc_context (gc_alloc_context* acontext, void* param)
6165 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
6166 g_theGCHeap->FixAllocContext(acontext, (void*)(size_t)(args->for_gc_p), args->heap);
6169 void gc_heap::fix_allocation_contexts (BOOL for_gc_p)
6171 fix_alloc_context_args args;
6172 args.for_gc_p = for_gc_p;
6175 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
6176 fix_youngest_allocation_area();
6177 fix_uoh_allocation_area();
6180 void gc_heap::fix_older_allocation_area (generation* older_gen)
6182 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
6183 if (generation_allocation_limit (older_gen) !=
6184 heap_segment_plan_allocated (older_gen_seg))
6186 uint8_t* point = generation_allocation_pointer (older_gen);
6188 size_t size = (generation_allocation_limit (older_gen) -
6189 generation_allocation_pointer (older_gen));
6192 assert ((size >= Align (min_obj_size)));
6193 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
6194 make_unused_array (point, size);
6195 if (size >= min_free_list)
6197 generation_allocator (older_gen)->thread_item_front (point, size);
6198 add_gen_free (older_gen->gen_num, size);
6199 generation_free_list_space (older_gen) += size;
6203 generation_free_obj_space (older_gen) += size;
6209 assert (older_gen_seg != ephemeral_heap_segment);
6210 heap_segment_plan_allocated (older_gen_seg) =
6211 generation_allocation_pointer (older_gen);
6212 generation_allocation_limit (older_gen) =
6213 generation_allocation_pointer (older_gen);
6216 generation_allocation_pointer (older_gen) = 0;
6217 generation_allocation_limit (older_gen) = 0;
6220 void gc_heap::set_allocation_heap_segment (generation* gen)
6222 uint8_t* p = generation_allocation_start (gen);
6224 heap_segment* seg = generation_allocation_segment (gen);
6225 if (in_range_for_segment (p, seg))
6228 // try ephemeral heap segment in case of heap expansion
6229 seg = ephemeral_heap_segment;
6230 if (!in_range_for_segment (p, seg))
6232 seg = heap_segment_rw (generation_start_segment (gen));
6234 PREFIX_ASSUME(seg != NULL);
6236 while (!in_range_for_segment (p, seg))
6238 seg = heap_segment_next_rw (seg);
6239 PREFIX_ASSUME(seg != NULL);
6243 generation_allocation_segment (gen) = seg;
6246 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6249 assert (Align ((size_t)start) == (size_t)start);
6250 generation_allocation_start (gen) = start;
6251 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6252 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6253 set_allocation_heap_segment (gen);
6256 bool gc_heap::new_allocation_allowed (int gen_number)
6258 #ifdef BACKGROUND_GC
6259 //TODO BACKGROUND_GC this is for test only
6260 if (!settings.allocations_allowed)
6262 dprintf (2, ("new allocation not allowed"));
6265 #endif //BACKGROUND_GC
6267 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6269 if (gen_number != 0)
6271 // For UOH we will give it more budget before we try a GC.
6272 if (settings.concurrent)
6274 dynamic_data* dd2 = dynamic_data_of (gen_number);
6276 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6284 #ifndef MULTIPLE_HEAPS
6285 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6287 dprintf (3, ("evaluating allocation rate"));
6288 dynamic_data* dd0 = dynamic_data_of (0);
6289 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6292 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6293 if ((ctime - allocation_running_time) > 1000)
6295 dprintf (2, (">1s since last gen0 gc"));
6300 allocation_running_amount = dd_new_allocation (dd0);
6304 #endif //MULTIPLE_HEAPS
6309 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6311 return dd_desired_allocation (dynamic_data_of (gen_number));
6315 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6317 return dd_new_allocation (dynamic_data_of (gen_number));
6320 //return the amount allocated so far in gen_number
6322 ptrdiff_t gc_heap::get_allocation (int gen_number)
6324 dynamic_data* dd = dynamic_data_of (gen_number);
6326 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6330 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6332 size_t new_size = max (init_len, 2*len);
6333 mark* tmp = new (nothrow) mark [new_size];
6336 memcpy (tmp, m, len * sizeof (mark));
6344 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6350 uint8_t* pinned_plug (mark* m)
6356 size_t& pinned_len (mark* m)
6362 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6364 m->len = pinned_plug (m) - pin_free_space_start;
6366 m->allocation_context_start_region = pin_free_space_start;
6367 #endif //SHORT_PLUGS
6372 uint8_t*& pin_allocation_context_start_region (mark* m)
6374 return m->allocation_context_start_region;
6377 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6379 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6380 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6381 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6382 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6383 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6384 return plug_start_in_saved;
6388 void set_padding_in_expand (uint8_t* old_loc,
6389 BOOL set_padding_on_saved_p,
6390 mark* pinned_plug_entry)
6392 if (set_padding_on_saved_p)
6394 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6398 set_plug_padded (old_loc);
6403 void clear_padding_in_expand (uint8_t* old_loc,
6404 BOOL set_padding_on_saved_p,
6405 mark* pinned_plug_entry)
6407 if (set_padding_on_saved_p)
6409 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6413 clear_plug_padded (old_loc);
6416 #endif //SHORT_PLUGS
6418 void gc_heap::reset_pinned_queue()
6424 void gc_heap::reset_pinned_queue_bos()
6429 // last_pinned_plug is only for asserting purpose.
6430 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6432 if (last_pinned_plug)
6434 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6435 assert (last_pinned_plug == last_m.first);
6436 if (last_m.saved_post_p)
6438 last_m.saved_post_p = FALSE;
6439 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6440 // We need to recover what the gap has overwritten.
6441 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6443 last_m.len += plug_size;
6444 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6448 void gc_heap::set_allocator_next_pin (generation* gen)
6450 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6451 if (!(pinned_plug_que_empty_p()))
6453 mark* oldest_entry = oldest_pin();
6454 uint8_t* plug = pinned_plug (oldest_entry);
6455 if ((plug >= generation_allocation_pointer (gen)) &&
6456 (plug < generation_allocation_limit (gen)))
6458 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6459 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6461 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6462 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6465 assert (!((plug < generation_allocation_pointer (gen)) &&
6466 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6470 // After we set the info, we increase tos.
6471 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6474 UNREFERENCED_PARAMETER(last_pinned_plug);
6477 mark& m = mark_stack_array[mark_stack_tos];
6478 assert (m.first == last_pinned_plug);
6483 // Why are we checking here? gen is never 0.
6486 set_allocator_next_pin (gen);
6490 size_t gc_heap::deque_pinned_plug ()
6492 dprintf (3, ("deque: %Id", mark_stack_bos));
6493 size_t m = mark_stack_bos;
6499 mark* gc_heap::pinned_plug_of (size_t bos)
6501 return &mark_stack_array [ bos ];
6505 mark* gc_heap::oldest_pin ()
6507 return pinned_plug_of (mark_stack_bos);
6511 BOOL gc_heap::pinned_plug_que_empty_p ()
6513 return (mark_stack_bos == mark_stack_tos);
6517 mark* gc_heap::before_oldest_pin()
6519 if (mark_stack_bos >= 1)
6520 return pinned_plug_of (mark_stack_bos-1);
6526 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6528 return ((o >= ephemeral_low) && (o < ephemeral_high));
6533 int& gc_heap::mark_stack_busy()
6535 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6539 void gc_heap::make_mark_stack (mark* arr)
6541 reset_pinned_queue();
6542 mark_stack_array = arr;
6543 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6545 mark_stack_busy() = 0;
6549 #ifdef BACKGROUND_GC
6551 size_t& gc_heap::bpromoted_bytes(int thread)
6553 #ifdef MULTIPLE_HEAPS
6554 return g_bpromoted [thread*16];
6555 #else //MULTIPLE_HEAPS
6556 UNREFERENCED_PARAMETER(thread);
6558 #endif //MULTIPLE_HEAPS
6561 void gc_heap::make_background_mark_stack (uint8_t** arr)
6563 background_mark_stack_array = arr;
6564 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6565 background_mark_stack_tos = arr;
6568 void gc_heap::make_c_mark_list (uint8_t** arr)
6571 c_mark_list_index = 0;
6572 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6574 #endif //BACKGROUND_GC
6579 // The card bundle keeps track of groups of card words.
6580 static const size_t card_bundle_word_width = 32;
6582 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6583 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6586 size_t card_bundle_word (size_t cardb)
6588 return cardb / card_bundle_word_width;
6592 uint32_t card_bundle_bit (size_t cardb)
6594 return (uint32_t)(cardb % card_bundle_word_width);
6597 size_t align_cardw_on_bundle (size_t cardw)
6599 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6602 // Get the card bundle representing a card word
6603 size_t cardw_card_bundle (size_t cardw)
6605 return cardw / card_bundle_size;
6608 // Get the first card word in a card bundle
6609 size_t card_bundle_cardw (size_t cardb)
6611 return cardb * card_bundle_size;
6614 // Clear the specified card bundle
6615 void gc_heap::card_bundle_clear (size_t cardb)
6617 uint32_t bit = (uint32_t)(1 << card_bundle_bit (cardb));
6618 uint32_t* bundle = &card_bundle_table[card_bundle_word (cardb)];
6619 #ifdef MULTIPLE_HEAPS
6620 // card bundles may straddle segments and heaps, thus bits may be cleared concurrently
6621 if ((*bundle & bit) != 0)
6623 Interlocked::And (bundle, ~bit);
6630 assert ((*bundle & bit) == 0);
6632 dprintf (2, ("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6633 (size_t)card_bundle_cardw (cardb+1)));
6636 inline void set_bundle_bits (uint32_t* bundle, uint32_t bits)
6638 #ifdef MULTIPLE_HEAPS
6639 // card bundles may straddle segments and heaps, thus bits may be set concurrently
6640 if ((*bundle & bits) != bits)
6642 Interlocked::Or (bundle, bits);
6649 assert ((*bundle & bits) == bits);
6652 void gc_heap::card_bundle_set (size_t cardb)
6654 uint32_t bits = (1 << card_bundle_bit (cardb));
6655 set_bundle_bits (&card_bundle_table [card_bundle_word (cardb)], bits);
6658 // Set the card bundle bits between start_cardb and end_cardb
6659 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6661 if (start_cardb == end_cardb)
6663 card_bundle_set(start_cardb);
6667 size_t start_word = card_bundle_word (start_cardb);
6668 size_t end_word = card_bundle_word (end_cardb);
6670 if (start_word < end_word)
6672 // Set the partial words
6673 uint32_t bits = highbits (~0u, card_bundle_bit (start_cardb));
6674 set_bundle_bits (&card_bundle_table [start_word], bits);
6676 if (card_bundle_bit (end_cardb))
6678 bits = lowbits (~0u, card_bundle_bit (end_cardb));
6679 set_bundle_bits (&card_bundle_table [end_word], bits);
6682 // Set the full words
6683 for (size_t i = start_word + 1; i < end_word; i++)
6685 card_bundle_table [i] = ~0u;
6690 uint32_t bits = (highbits (~0u, card_bundle_bit (start_cardb)) &
6691 lowbits (~0u, card_bundle_bit (end_cardb)));
6692 set_bundle_bits (&card_bundle_table [start_word], bits);
6696 // Indicates whether the specified bundle is set.
6697 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6699 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6702 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6703 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6705 // Number of heap bytes represented by a card bundle word
6706 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6708 // Align the start of the region down
6709 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6711 // Align the end of the region up
6712 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6714 // Make sure they're really aligned
6715 assert (((size_t)from & (cbw_span - 1)) == 0);
6716 assert (((size_t)end & (cbw_span - 1)) == 0);
6718 return ((end - from) / cbw_span) * sizeof (uint32_t);
6721 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6722 // where a theoretical card bundle table that represents every address (starting from 0) would
6723 // start if the bundle word representing the address were to be located at the pointer passed in.
6724 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6725 // for a given address is using a simple shift operation on the address.
6726 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6728 // The number of bytes of heap memory represented by a card bundle word
6729 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6731 // Each card bundle word is 32 bits
6732 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6735 void gc_heap::enable_card_bundles ()
6737 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6739 dprintf (1, ("Enabling card bundles"));
6741 // We initially set all of the card bundles
6742 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6743 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6744 settings.card_bundles = TRUE;
6748 BOOL gc_heap::card_bundles_enabled ()
6750 return settings.card_bundles;
6753 #endif // CARD_BUNDLE
6755 #if defined (TARGET_AMD64)
6756 #define brick_size ((size_t)4096)
6758 #define brick_size ((size_t)2048)
6759 #endif //TARGET_AMD64
6762 size_t gc_heap::brick_of (uint8_t* add)
6764 return (size_t)(add - lowest_address) / brick_size;
6768 uint8_t* gc_heap::brick_address (size_t brick)
6770 return lowest_address + (brick_size * brick);
6774 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6776 for (size_t i = brick_of (from);i < brick_of (end); i++)
6780 //codes for the brick entries:
6781 //entry == 0 -> not assigned
6782 //entry >0 offset is entry-1
6783 //entry <0 jump back entry bricks
6787 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6793 assert (val < 32767);
6795 brick_table [index] = (short)val+1;
6797 brick_table [index] = (short)val;
6801 int gc_heap::get_brick_entry (size_t index)
6803 #ifdef MULTIPLE_HEAPS
6804 return VolatileLoadWithoutBarrier(&brick_table [index]);
6806 return brick_table[index];
6812 uint8_t* align_on_brick (uint8_t* add)
6814 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6818 uint8_t* align_lower_brick (uint8_t* add)
6820 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6823 size_t size_brick_of (uint8_t* from, uint8_t* end)
6825 assert (((size_t)from & (brick_size-1)) == 0);
6826 assert (((size_t)end & (brick_size-1)) == 0);
6828 return ((end - from) / brick_size) * sizeof (short);
6832 uint8_t* gc_heap::card_address (size_t card)
6834 return (uint8_t*) (card_size * card);
6838 size_t gc_heap::card_of ( uint8_t* object)
6840 return (size_t)(object) / card_size;
6844 size_t gc_heap::card_to_brick (size_t card)
6846 return brick_of (card_address (card));
6850 uint8_t* align_on_card (uint8_t* add)
6852 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6855 uint8_t* align_on_card_word (uint8_t* add)
6857 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6861 uint8_t* align_lower_card (uint8_t* add)
6863 return (uint8_t*)((size_t)add & ~(card_size-1));
6867 void gc_heap::clear_card (size_t card)
6869 card_table [card_word (card)] =
6870 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6871 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6872 (size_t)card_address (card+1)));
6876 void gc_heap::set_card (size_t card)
6878 size_t word = card_word (card);
6879 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6881 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6882 // Also set the card bundle that corresponds to the card
6883 size_t bundle_to_set = cardw_card_bundle(word);
6885 card_bundle_set(bundle_to_set);
6887 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6892 BOOL gc_heap::card_set_p (size_t card)
6894 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6897 // Returns the number of DWORDs in the card table that cover the
6898 // range of addresses [from, end[.
6899 size_t count_card_of (uint8_t* from, uint8_t* end)
6901 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6904 // Returns the number of bytes to allocate for a card table
6905 // that covers the range of addresses [from, end[.
6906 size_t size_card_of (uint8_t* from, uint8_t* end)
6908 return count_card_of (from, end) * sizeof(uint32_t);
6911 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6912 class card_table_info
6916 uint8_t* lowest_address;
6917 uint8_t* highest_address;
6921 uint32_t* card_bundle_table;
6922 #endif //CARD_BUNDLE
6924 // mark_array is always at the end of the data structure because we
6925 // want to be able to make one commit call for everything before it.
6926 #ifdef BACKGROUND_GC
6927 uint32_t* mark_array;
6928 #endif //BACKGROUND_GC
6931 uint32_t* next_card_table;
6934 //These are accessors on untranslated cardtable
6936 unsigned& card_table_refcount (uint32_t* c_table)
6938 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6942 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6944 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6947 uint32_t* translate_card_table (uint32_t* ct)
6949 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6953 uint8_t*& card_table_highest_address (uint32_t* c_table)
6955 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6959 short*& card_table_brick_table (uint32_t* c_table)
6961 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6966 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6968 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6970 #endif //CARD_BUNDLE
6972 #ifdef BACKGROUND_GC
6974 uint32_t*& card_table_mark_array (uint32_t* c_table)
6976 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6980 #define mark_bit_pitch ((size_t)16)
6982 #define mark_bit_pitch ((size_t)8)
6983 #endif // HOST_64BIT
6984 #define mark_word_width ((size_t)32)
6985 #define mark_word_size (mark_word_width * mark_bit_pitch)
6988 uint8_t* align_on_mark_bit (uint8_t* add)
6990 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6994 uint8_t* align_lower_mark_bit (uint8_t* add)
6996 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
7000 BOOL is_aligned_on_mark_word (uint8_t* add)
7002 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
7006 uint8_t* align_on_mark_word (uint8_t* add)
7008 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
7012 uint8_t* align_lower_mark_word (uint8_t* add)
7014 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
7018 size_t mark_bit_of (uint8_t* add)
7020 return ((size_t)add / mark_bit_pitch);
7024 unsigned int mark_bit_bit (size_t mark_bit)
7026 return (unsigned int)(mark_bit % mark_word_width);
7030 size_t mark_bit_word (size_t mark_bit)
7032 return (mark_bit / mark_word_width);
7036 size_t mark_word_of (uint8_t* add)
7038 return ((size_t)add) / mark_word_size;
7041 uint8_t* mark_word_address (size_t wd)
7043 return (uint8_t*)(wd*mark_word_size);
7046 uint8_t* mark_bit_address (size_t mark_bit)
7048 return (uint8_t*)(mark_bit*mark_bit_pitch);
7052 size_t mark_bit_bit_of (uint8_t* add)
7054 return (((size_t)add / mark_bit_pitch) % mark_word_width);
7058 unsigned int gc_heap::mark_array_marked(uint8_t* add)
7060 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
7064 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
7066 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
7070 void gc_heap::mark_array_set_marked (uint8_t* add)
7072 size_t index = mark_word_of (add);
7073 uint32_t val = (1 << mark_bit_bit_of (add));
7074 #ifdef MULTIPLE_HEAPS
7075 Interlocked::Or (&(mark_array [index]), val);
7077 mark_array [index] |= val;
7082 void gc_heap::mark_array_clear_marked (uint8_t* add)
7084 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
7087 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
7089 assert (((size_t)from & ((mark_word_size)-1)) == 0);
7090 assert (((size_t)end & ((mark_word_size)-1)) == 0);
7091 return sizeof (uint32_t)*(((end - from) / mark_word_size));
7094 //In order to eliminate the lowest_address in the mark array
7095 //computations (mark_word_of, etc) mark_array is offset
7096 // according to the lowest_address.
7097 uint32_t* translate_mark_array (uint32_t* ma)
7099 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
7102 // from and end must be page aligned addresses.
7103 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
7104 #ifdef FEATURE_BASICFREEZE
7105 , BOOL read_only/*=FALSE*/
7106 #endif // FEATURE_BASICFREEZE
7109 if(!gc_can_use_concurrent)
7112 #ifdef FEATURE_BASICFREEZE
7114 #endif // FEATURE_BASICFREEZE
7116 assert (from == align_on_mark_word (from));
7118 assert (end == align_on_mark_word (end));
7120 #ifdef BACKGROUND_GC
7121 uint8_t* current_lowest_address = background_saved_lowest_address;
7122 uint8_t* current_highest_address = background_saved_highest_address;
7124 uint8_t* current_lowest_address = lowest_address;
7125 uint8_t* current_highest_address = highest_address;
7126 #endif //BACKGROUND_GC
7128 //there is a possibility of the addresses to be
7129 //outside of the covered range because of a newly allocated
7130 //large object segment
7131 if ((end <= current_highest_address) && (from >= current_lowest_address))
7133 size_t beg_word = mark_word_of (align_on_mark_word (from));
7134 //align end word to make sure to cover the address
7135 size_t end_word = mark_word_of (align_on_mark_word (end));
7136 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
7137 (size_t)mark_word_address (beg_word),
7138 (size_t)mark_word_address (end_word),
7139 (size_t)from, (size_t)end,
7140 (check_only ? "check_only" : "clear")));
7144 while (op < mark_word_address (beg_word))
7146 mark_array_clear_marked (op);
7147 op += mark_bit_pitch;
7150 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
7155 //Beware, it is assumed that the mark array word straddling
7156 //start has been cleared before
7157 //verify that the array is empty.
7158 size_t markw = mark_word_of (align_on_mark_word (from));
7159 size_t markw_end = mark_word_of (align_on_mark_word (end));
7160 while (markw < markw_end)
7162 assert (!(mark_array [markw]));
7165 uint8_t* p = mark_word_address (markw_end);
7168 assert (!(mark_array_marked (p)));
7175 #endif //BACKGROUND_GC
7177 //These work on untranslated card tables
7179 uint32_t*& card_table_next (uint32_t* c_table)
7181 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
7185 size_t& card_table_size (uint32_t* c_table)
7187 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
7190 void own_card_table (uint32_t* c_table)
7192 card_table_refcount (c_table) += 1;
7195 void destroy_card_table (uint32_t* c_table);
7197 void delete_next_card_table (uint32_t* c_table)
7199 uint32_t* n_table = card_table_next (c_table);
7202 if (card_table_next (n_table))
7204 delete_next_card_table (n_table);
7206 if (card_table_refcount (n_table) == 0)
7208 destroy_card_table (n_table);
7209 card_table_next (c_table) = 0;
7214 void release_card_table (uint32_t* c_table)
7216 assert (card_table_refcount (c_table) >0);
7217 card_table_refcount (c_table) -= 1;
7218 if (card_table_refcount (c_table) == 0)
7220 delete_next_card_table (c_table);
7221 if (card_table_next (c_table) == 0)
7223 destroy_card_table (c_table);
7224 // sever the link from the parent
7225 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7227 g_gc_card_table = 0;
7229 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7230 g_gc_card_bundle_table = 0;
7232 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7233 SoftwareWriteWatch::StaticClose();
7234 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7238 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7241 while (p_table && (card_table_next (p_table) != c_table))
7242 p_table = card_table_next (p_table);
7243 card_table_next (p_table) = 0;
7250 void destroy_card_table (uint32_t* c_table)
7252 // delete (uint32_t*)&card_table_refcount(c_table);
7254 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7255 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7258 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7260 assert (g_gc_lowest_address == start);
7261 assert (g_gc_highest_address == end);
7263 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7265 size_t bs = size_brick_of (start, end);
7266 size_t cs = size_card_of (start, end);
7267 #ifdef BACKGROUND_GC
7268 size_t ms = (gc_can_use_concurrent ?
7269 size_mark_array_of (start, end) :
7273 #endif //BACKGROUND_GC
7278 if (can_use_write_watch_for_card_table())
7280 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7281 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7282 // If we're not manually managing the card bundles, we will need to use OS write
7283 // watch APIs over this region to track changes.
7284 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7287 #endif //CARD_BUNDLE
7290 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7291 size_t sw_ww_table_offset = 0;
7292 if (gc_can_use_concurrent)
7294 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7295 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7296 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7298 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7300 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7301 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7302 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7304 st += (st_table_offset_aligned - st_table_offset);
7306 // it is impossible for alloc_size to overflow due bounds on each of
7308 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7309 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7314 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7315 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7317 // mark array will be committed separately (per segment).
7318 size_t commit_size = alloc_size - ms;
7320 if (!virtual_commit (mem, commit_size, gc_oh_num::none))
7322 dprintf (1, ("Card table commit failed"));
7323 GCToOSInterface::VirtualRelease (mem, alloc_size);
7327 // initialize the ref count
7328 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7329 card_table_refcount (ct) = 0;
7330 card_table_lowest_address (ct) = start;
7331 card_table_highest_address (ct) = end;
7332 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7333 card_table_size (ct) = alloc_size;
7334 card_table_next (ct) = 0;
7337 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7339 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7340 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7343 #endif //CARD_BUNDLE
7345 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7346 if (gc_can_use_concurrent)
7348 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7350 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7352 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7353 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7354 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7356 #ifdef BACKGROUND_GC
7357 if (gc_can_use_concurrent)
7358 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7360 card_table_mark_array (ct) = NULL;
7361 #endif //BACKGROUND_GC
7363 return translate_card_table(ct);
7366 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7368 #ifdef MULTIPLE_HEAPS
7369 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7371 gc_heap* hp = gc_heap::g_heaps [hn];
7372 hp->fgm_result.set_fgm (f, s, loh_p);
7374 #else //MULTIPLE_HEAPS
7375 fgm_result.set_fgm (f, s, loh_p);
7376 #endif //MULTIPLE_HEAPS
7379 //returns 0 for success, -1 otherwise
7380 // We are doing all the decommitting here because we want to make sure we have
7381 // enough memory to do so - if we do this during copy_brick_card_table and
7382 // and fail to decommit it would make the failure case very complicated to
7383 // handle. This way we can waste some decommit if we call this multiple
7384 // times before the next FGC but it's easier to handle the failure case.
7385 int gc_heap::grow_brick_card_tables (uint8_t* start,
7388 heap_segment* new_seg,
7392 uint8_t* la = g_gc_lowest_address;
7393 uint8_t* ha = g_gc_highest_address;
7394 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7395 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7396 seg_mapping* new_seg_mapping_table = nullptr;
7397 #ifdef BACKGROUND_GC
7398 // This value is only for logging purpose - it's not necessarily exactly what we
7399 // would commit for mark array but close enough for diagnostics purpose.
7400 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7401 #endif //BACKGROUND_GC
7403 // See if the address is already covered
7404 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7407 //modify the highest address so the span covered
7408 //is twice the previous one.
7409 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7410 // On non-Windows systems, we get only an approximate value that can possibly be
7411 // slightly lower than the saved_g_highest_address.
7412 // In such case, we set the top to the saved_g_highest_address so that the
7413 // card and brick tables always cover the whole new range.
7414 if (top < saved_g_highest_address)
7416 top = saved_g_highest_address;
7420 if (ps > (uint64_t)200*1024*1024*1024)
7421 ps += (uint64_t)100*1024*1024*1024;
7423 #endif // HOST_64BIT
7426 if (saved_g_lowest_address < g_gc_lowest_address)
7428 if (ps > (size_t)g_gc_lowest_address)
7429 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7432 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7433 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7437 if (saved_g_highest_address > g_gc_highest_address)
7439 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7440 if (saved_g_highest_address > top)
7441 saved_g_highest_address = top;
7444 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7445 (size_t)saved_g_lowest_address,
7446 (size_t)saved_g_highest_address));
7448 bool write_barrier_updated = false;
7449 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7450 uint32_t* saved_g_card_table = g_gc_card_table;
7452 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7453 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7457 uint32_t* translated_ct = 0;
7460 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7461 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7463 #ifdef BACKGROUND_GC
7464 size_t ms = (gc_heap::gc_can_use_concurrent ?
7465 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7469 #endif //BACKGROUND_GC
7474 if (can_use_write_watch_for_card_table())
7476 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7478 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7479 // If we're not manually managing the card bundles, we will need to use OS write
7480 // watch APIs over this region to track changes.
7481 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7484 #endif //CARD_BUNDLE
7487 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7488 size_t sw_ww_table_offset = 0;
7489 if (gc_can_use_concurrent)
7491 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7492 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7494 sw_ww_table_offset -
7495 sw_ww_size_before_table +
7496 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7498 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7500 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7501 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7502 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7503 st += (st_table_offset_aligned - st_table_offset);
7505 // it is impossible for alloc_size to overflow due bounds on each of
7507 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7508 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7509 cs, bs, cb, wws, st, ms));
7511 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7515 set_fgm_result (fgm_grow_table, alloc_size, uoh_p);
7519 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7520 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7523 // mark array will be committed separately (per segment).
7524 size_t commit_size = alloc_size - ms;
7526 if (!virtual_commit (mem, commit_size, gc_oh_num::none))
7528 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7529 set_fgm_result (fgm_commit_table, commit_size, uoh_p);
7534 ct = (uint32_t*)(mem + sizeof (card_table_info));
7535 card_table_refcount (ct) = 0;
7536 card_table_lowest_address (ct) = saved_g_lowest_address;
7537 card_table_highest_address (ct) = saved_g_highest_address;
7538 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7540 //clear the card table
7542 memclr ((uint8_t*)ct,
7543 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7544 (card_size * card_word_width))
7545 + sizeof (uint32_t)));
7548 bt = (short*)((uint8_t*)ct + cs);
7550 // No initialization needed, will be done in copy_brick_card
7552 card_table_brick_table (ct) = bt;
7555 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7556 //set all bundle to look at all of the cards
7557 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7558 #endif //CARD_BUNDLE
7560 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7561 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7562 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7563 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7564 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7565 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7567 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7568 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7569 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7570 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7571 // if an OOM occurs.
7573 #ifdef BACKGROUND_GC
7574 if(gc_can_use_concurrent)
7575 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7577 card_table_mark_array (ct) = NULL;
7578 #endif //BACKGROUND_GC
7580 translated_ct = translate_card_table (ct);
7582 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7583 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7585 #ifdef BACKGROUND_GC
7586 if (hp->should_commit_mark_array())
7588 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7589 saved_g_lowest_address, saved_g_highest_address,
7590 card_table_mark_array (ct),
7591 translate_mark_array (card_table_mark_array (ct))));
7592 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7593 if (!commit_new_mark_array_global (new_mark_array))
7595 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7596 set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p);
7600 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7602 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7603 set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p);
7609 clear_commit_flag_global();
7611 #endif //BACKGROUND_GC
7613 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7614 if (gc_can_use_concurrent)
7616 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7617 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7618 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7619 // table info lazily as done for card tables.
7621 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7622 // from a GC thread which means we are in a blocking GC and also suspended.
7623 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7624 if (!is_runtime_suspended)
7626 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7627 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7628 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7629 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7630 // g_gc_highest_address.
7634 g_gc_card_table = translated_ct;
7636 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7637 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7640 SoftwareWriteWatch::SetResizedUntranslatedTable(
7641 mem + sw_ww_table_offset,
7642 saved_g_lowest_address,
7643 saved_g_highest_address);
7645 seg_mapping_table = new_seg_mapping_table;
7647 // Since the runtime is already suspended, update the write barrier here as well.
7648 // This passes a bool telling whether we need to switch to the post
7649 // grow version of the write barrier. This test tells us if the new
7650 // segment was allocated at a lower address than the old, requiring
7651 // that we start doing an upper bounds check in the write barrier.
7652 g_gc_lowest_address = saved_g_lowest_address;
7653 g_gc_highest_address = saved_g_highest_address;
7654 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7655 write_barrier_updated = true;
7657 if (!is_runtime_suspended)
7663 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7665 g_gc_card_table = translated_ct;
7667 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7668 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7672 if (!write_barrier_updated)
7674 seg_mapping_table = new_seg_mapping_table;
7675 GCToOSInterface::FlushProcessWriteBuffers();
7676 g_gc_lowest_address = saved_g_lowest_address;
7677 g_gc_highest_address = saved_g_highest_address;
7679 // This passes a bool telling whether we need to switch to the post
7680 // grow version of the write barrier. This test tells us if the new
7681 // segment was allocated at a lower address than the old, requiring
7682 // that we start doing an upper bounds check in the write barrier.
7683 // This will also suspend the runtime if the write barrier type needs
7684 // to be changed, so we are doing this after all global state has
7685 // been updated. See the comment above suspend_EE() above for more
7687 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7695 assert(g_gc_card_table == saved_g_card_table);
7697 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7698 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7701 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7702 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7704 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7705 assert (!"release failed");
7713 #ifdef BACKGROUND_GC
7714 if (hp->should_commit_mark_array())
7716 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7717 if (!commit_mark_array_new_seg (hp, new_seg))
7719 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7720 set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p);
7724 #endif //BACKGROUND_GC
7730 //copy all of the arrays managed by the card table for a page aligned range
7731 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7732 short* old_brick_table,
7733 uint8_t* start, uint8_t* end)
7735 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7738 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7741 short* brick_start = &brick_table [brick_of (start)];
7742 if (old_brick_table)
7744 // segments are always on page boundaries
7745 memcpy (brick_start, &old_brick_table[brick_offset],
7746 size_brick_of (start, end));
7751 // This is a large heap, just clear the brick table
7754 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7756 if (gc_heap::background_running_p())
7758 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7760 // We don't need to go through all the card tables here because
7761 // we only need to copy from the GC version of the mark array - when we
7762 // mark (even in allocate_uoh_object) we always use that mark array.
7763 if ((card_table_highest_address (old_ct) >= start) &&
7764 (card_table_lowest_address (old_ct) <= end))
7766 if ((background_saved_highest_address >= start) &&
7767 (background_saved_lowest_address <= end))
7769 //copy the mark bits
7770 // segments are always on page boundaries
7771 uint8_t* m_start = max (background_saved_lowest_address, start);
7772 uint8_t* m_end = min (background_saved_highest_address, end);
7773 memcpy (&mark_array[mark_word_of (m_start)],
7774 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7775 size_mark_array_of (m_start, m_end));
7780 //only large segments can be out of range
7781 assert (old_brick_table == 0);
7785 // n way merge with all of the card table ever used in between
7786 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7789 while (card_table_next (old_ct) != ct)
7791 //copy if old card table contained [start, end[
7792 if ((card_table_highest_address (ct) >= end) &&
7793 (card_table_lowest_address (ct) <= start))
7795 // or the card_tables
7797 size_t start_word = card_word (card_of (start));
7799 uint32_t* dest = &card_table[start_word];
7800 uint32_t* src = &((translate_card_table (ct))[start_word]);
7801 ptrdiff_t count = count_card_of (start, end);
7802 for (int x = 0; x < count; x++)
7806 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7809 card_bundle_set(cardw_card_bundle(start_word+x));
7817 ct = card_table_next (ct);
7821 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7822 void gc_heap::init_brick_card_range (heap_segment* seg)
7824 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7825 (size_t)heap_segment_mem (seg),
7826 (size_t)heap_segment_allocated (seg)));
7828 // initialize the brick table
7829 for (size_t b = brick_of (heap_segment_mem (seg));
7830 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7836 #ifdef BACKGROUND_GC
7837 if (gc_heap::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7840 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7842 #endif //BACKGROUND_GC
7844 clear_card_for_addresses (heap_segment_mem (seg),
7845 heap_segment_allocated (seg));
7848 void gc_heap::copy_brick_card_table()
7850 uint32_t* old_card_table = card_table;
7851 short* old_brick_table = brick_table;
7853 uint8_t* la = lowest_address;
7855 uint8_t* ha = highest_address;
7856 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7857 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7860 /* todo: Need a global lock for this */
7861 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7862 own_card_table (ct);
7863 card_table = translate_card_table (ct);
7864 /* End of global lock */
7865 highest_address = card_table_highest_address (ct);
7866 lowest_address = card_table_lowest_address (ct);
7868 brick_table = card_table_brick_table (ct);
7870 #ifdef BACKGROUND_GC
7871 if (gc_can_use_concurrent)
7873 mark_array = translate_mark_array (card_table_mark_array (ct));
7874 assert (mark_word_of (g_gc_highest_address) ==
7875 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7879 #endif //BACKGROUND_GC
7882 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7884 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7885 // start of the untranslated table.
7886 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7887 card_table_card_bundle_table (ct));
7889 //set the card table if we are in a heap growth scenario
7890 if (card_bundles_enabled())
7892 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7893 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7895 //check if we need to turn on card_bundles.
7896 #ifdef MULTIPLE_HEAPS
7897 // use INT64 arithmetic here because of possible overflow on 32p
7898 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7900 // use INT64 arithmetic here because of possible overflow on 32p
7901 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7902 #endif //MULTIPLE_HEAPS
7903 if (reserved_memory >= th)
7905 enable_card_bundles();
7908 #endif //CARD_BUNDLE
7910 // for each of the segments and heaps, copy the brick table and
7911 // or the card table
7912 for (int i = max_generation; i < total_generation_count; i++)
7914 heap_segment* seg = generation_start_segment (generation_of (i));
7917 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7919 //check if it became in range
7920 if ((heap_segment_reserved (seg) > lowest_address) &&
7921 (heap_segment_mem (seg) < highest_address))
7923 set_ro_segment_in_range (seg);
7928 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7929 copy_brick_card_range (la, old_card_table,
7930 (i < uoh_start_generation) ? old_brick_table : NULL,
7931 align_lower_page (heap_segment_mem (seg)),
7934 seg = heap_segment_next (seg);
7938 release_card_table (&old_card_table[card_word (card_of(la))]);
7941 #ifdef FEATURE_BASICFREEZE
7942 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7944 enter_spin_lock (&gc_heap::gc_lock);
7946 if (!gc_heap::seg_table->ensure_space_for_insert ()
7947 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7949 leave_spin_lock(&gc_heap::gc_lock);
7953 //insert at the head of the segment list
7954 generation* gen2 = generation_of (max_generation);
7955 heap_segment* oldhead = generation_start_segment (gen2);
7956 heap_segment_next (seg) = oldhead;
7957 generation_start_segment (gen2) = seg;
7959 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7961 seg_mapping_table_add_ro_segment (seg);
7963 if ((heap_segment_reserved (seg) > lowest_address) &&
7964 (heap_segment_mem (seg) < highest_address))
7966 set_ro_segment_in_range (seg);
7969 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7971 leave_spin_lock (&gc_heap::gc_lock);
7975 // No one is calling this function right now. If this is getting called we need
7976 // to take care of decommitting the mark array for it - we will need to remember
7977 // which portion of the mark array was committed and only decommit that.
7978 void gc_heap::remove_ro_segment (heap_segment* seg)
7980 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7981 #ifdef BACKGROUND_GC
7982 if (gc_can_use_concurrent)
7984 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7985 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7986 false); // read_only segments need the mark clear
7988 #endif //BACKGROUND_GC
7990 enter_spin_lock (&gc_heap::gc_lock);
7992 seg_table->remove ((uint8_t*)seg);
7993 seg_mapping_table_remove_ro_segment (seg);
7995 // Locate segment (and previous segment) in the list.
7996 generation* gen2 = generation_of (max_generation);
7997 heap_segment* curr_seg = generation_start_segment (gen2);
7998 heap_segment* prev_seg = NULL;
8000 while (curr_seg && curr_seg != seg)
8002 prev_seg = curr_seg;
8003 curr_seg = heap_segment_next (curr_seg);
8005 assert (curr_seg == seg);
8007 // Patch previous segment (or list head if there is none) to skip the removed segment.
8009 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
8011 generation_start_segment (gen2) = heap_segment_next (curr_seg);
8013 leave_spin_lock (&gc_heap::gc_lock);
8015 #endif //FEATURE_BASICFREEZE
8017 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
8019 seg->flags |= heap_segment_flags_inrange;
8020 ro_segments_in_range = TRUE;
8026 uint8_t** make_mark_list (size_t size)
8028 uint8_t** mark_list = new (nothrow) uint8_t* [size];
8032 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
8034 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
8038 for (i = low+1; i <= high; i++)
8047 #ifndef USE_INTROSORT
8048 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
8050 if (((low + 16) >= high) || (depth > 100))
8054 for (i = low+1; i <= high; i++)
8057 for (j=i;j >low && val<*(j-1);j--)
8066 uint8_t *pivot, **left, **right;
8068 //sort low middle and high
8069 if (*(low+((high-low)/2)) < *low)
8070 swap (*(low+((high-low)/2)), *low);
8073 if (*high < *(low+((high-low)/2)))
8074 swap (*(low+((high-low)/2)), *high);
8076 swap (*(low+((high-low)/2)), *(high-1));
8078 left = low; right = high-1;
8080 while (*(--right) > pivot);
8081 while (*(++left) < pivot);
8084 swap(*left, *right);
8089 swap (*left, *(high-1));
8090 qsort1(low, left-1, depth+1);
8091 qsort1(left+1, high, depth+1);
8094 #endif //USE_INTROSORT
8095 void rqsort1( uint8_t* *low, uint8_t* *high)
8097 if ((low + 16) >= high)
8101 for (i = low+1; i <= high; i++)
8104 for (j=i;j >low && val>*(j-1);j--)
8113 uint8_t *pivot, **left, **right;
8115 //sort low middle and high
8116 if (*(low+((high-low)/2)) > *low)
8117 swap (*(low+((high-low)/2)), *low);
8120 if (*high > *(low+((high-low)/2)))
8121 swap (*(low+((high-low)/2)), *high);
8123 swap (*(low+((high-low)/2)), *(high-1));
8125 left = low; right = high-1;
8127 while (*(--right) < pivot);
8128 while (*(++left) > pivot);
8131 swap(*left, *right);
8136 swap (*left, *(high-1));
8137 rqsort1(low, left-1);
8138 rqsort1(left+1, high);
8142 // vxsort uses introsort as a fallback if the AVX2 instruction set is not supported
8143 #if defined(USE_INTROSORT) || defined(USE_VXSORT)
8148 static const int size_threshold = 64;
8149 static const int max_depth = 100;
8152 inline static void swap_elements(uint8_t** i,uint8_t** j)
8160 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8163 introsort_loop (begin, end, max_depth);
8164 insertionsort (begin, end);
8169 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8171 while (hi-lo >= size_threshold)
8173 if (depth_limit == 0)
8178 uint8_t** p=median_partition (lo, hi);
8179 depth_limit=depth_limit-1;
8180 introsort_loop (p, hi, depth_limit);
8185 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8187 uint8_t *pivot, **left, **right;
8189 //sort low middle and high
8190 if (*(low+((high-low)/2)) < *low)
8191 swap_elements ((low+((high-low)/2)), low);
8193 swap_elements (low, high);
8194 if (*high < *(low+((high-low)/2)))
8195 swap_elements ((low+((high-low)/2)), high);
8197 swap_elements ((low+((high-low)/2)), (high-1));
8199 left = low; right = high-1;
8201 while (*(--right) > pivot);
8202 while (*(++left) < pivot);
8205 swap_elements(left, right);
8210 swap_elements (left, (high-1));
8215 static void insertionsort (uint8_t** lo, uint8_t** hi)
8217 for (uint8_t** i=lo+1; i <= hi; i++)
8221 while((j > lo) && (t <*(j-1)))
8230 static void heapsort (uint8_t** lo, uint8_t** hi)
8232 size_t n = hi - lo + 1;
8233 for (size_t i=n / 2; i >= 1; i--)
8237 for (size_t i = n; i > 1; i--)
8239 swap_elements (lo, lo + i - 1);
8240 downheap(1, i - 1, lo);
8244 static void downheap (size_t i, size_t n, uint8_t** lo)
8246 uint8_t* d = *(lo + i - 1);
8251 if (child < n && *(lo + child - 1)<(*(lo + child)))
8255 if (!(d<*(lo + child - 1)))
8259 *(lo + i - 1) = *(lo + child - 1);
8267 #endif //defined(USE_INTROSORT) || defined(USE_VXSORT)
8270 static void do_vxsort (uint8_t** item_array, ptrdiff_t item_count, uint8_t* range_low, uint8_t* range_high)
8272 // above this threshold, using AVX2 for sorting will likely pay off
8273 // despite possible downclocking on some devices
8274 const size_t AVX2_THRESHOLD_SIZE = 8 * 1024;
8276 // above this threshold, using AVX51F for sorting will likely pay off
8277 // despite possible downclocking on current devices
8278 const size_t AVX512F_THRESHOLD_SIZE = 128 * 1024;
8280 if (item_count <= 1)
8283 if (IsSupportedInstructionSet (InstructionSet::AVX2) && (item_count > AVX2_THRESHOLD_SIZE))
8285 // is the range small enough for a 32-bit sort?
8286 // the 32-bit sort is almost twice as fast
8287 ptrdiff_t range = range_high - range_low;
8288 assert(sizeof(uint8_t*) == (1 << 3));
8289 ptrdiff_t scaled_range = range >> 3;
8290 if ((uint32_t)scaled_range == scaled_range)
8292 dprintf (3, ("Sorting mark lists as 32-bit offsets"));
8294 do_pack_avx2 (item_array, item_count, range_low);
8296 int32_t* item_array_32 = (int32_t*)item_array;
8298 // use AVX512F only if the list is large enough to pay for downclocking impact
8299 if (IsSupportedInstructionSet (InstructionSet::AVX512F) && (item_count > AVX512F_THRESHOLD_SIZE))
8301 do_vxsort_avx512 (item_array_32, &item_array_32[item_count - 1]);
8305 do_vxsort_avx2 (item_array_32, &item_array_32[item_count - 1]);
8308 do_unpack_avx2 (item_array_32, item_count, range_low);
8312 dprintf(3, ("Sorting mark lists"));
8314 // use AVX512F only if the list is large enough to pay for downclocking impact
8315 if (IsSupportedInstructionSet (InstructionSet::AVX512F) && (item_count > AVX512F_THRESHOLD_SIZE))
8317 do_vxsort_avx512 (item_array, &item_array[item_count - 1]);
8321 do_vxsort_avx2 (item_array, &item_array[item_count - 1]);
8327 dprintf (3, ("Sorting mark lists"));
8328 introsort::sort (item_array, &item_array[item_count - 1], 0);
8331 // check the array is sorted
8332 for (ptrdiff_t i = 0; i < item_count - 1; i++)
8334 assert (item_array[i] <= item_array[i + 1]);
8336 // check that the ends of the array are indeed in range
8337 // together with the above this implies all elements are in range
8338 assert ((range_low <= item_array[0]) && (item_array[item_count - 1] <= range_high));
8343 #ifdef MULTIPLE_HEAPS
8344 #ifdef PARALLEL_MARK_LIST_SORT
8346 void gc_heap::sort_mark_list()
8348 if (settings.condemned_generation >= max_generation)
8353 // if this heap had a mark list overflow, we don't do anything
8354 if (mark_list_index > mark_list_end)
8356 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8357 mark_list_overflow = true;
8361 #ifdef BACKGROUND_GC
8362 // we are not going to use the mark list if background GC is running
8363 // so let's not waste time sorting it
8364 if (gc_heap::background_running_p())
8366 mark_list_index = mark_list_end + 1;
8369 #endif //BACKGROUND_GC
8371 // if any other heap had a mark list overflow, we fake one too,
8372 // so we don't use an incomplete mark list by mistake
8373 for (int i = 0; i < n_heaps; i++)
8375 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8377 mark_list_index = mark_list_end + 1;
8378 // printf("sort_mark_list: overflow on heap %d\n", i);
8383 // unsigned long start = GetCycleCount32();
8385 // compute total mark list size and total ephemeral size
8386 size_t total_mark_list_size = 0;
8387 size_t total_ephemeral_size = 0;
8388 uint8_t* low = (uint8_t*)~0;
8390 for (int i = 0; i < n_heaps; i++)
8392 gc_heap* hp = g_heaps[i];
8393 size_t ephemeral_size = heap_segment_allocated (hp->ephemeral_heap_segment) - hp->gc_low;
8394 total_ephemeral_size += ephemeral_size;
8395 total_mark_list_size += (hp->mark_list_index - hp->mark_list);
8396 low = min (low, hp->gc_low);
8397 high = max (high, heap_segment_allocated (hp->ephemeral_heap_segment));
8400 // give up if this is not an ephemeral GC or the mark list size is unreasonably large
8401 if (total_mark_list_size > (total_ephemeral_size / 256))
8403 mark_list_index = mark_list_end + 1;
8404 // let's not count this as a mark list overflow
8405 mark_list_overflow = false;
8410 ptrdiff_t item_count = mark_list_index - mark_list;
8411 //#define WRITE_SORT_DATA
8412 #if defined(_DEBUG) || defined(WRITE_SORT_DATA)
8413 // in debug, make a copy of the mark list
8414 // for checking and debugging purposes
8415 uint8_t** mark_list_copy = &g_mark_list_copy[heap_number * mark_list_size];
8416 uint8_t** mark_list_copy_index = &mark_list_copy[item_count];
8417 for (ptrdiff_t i = 0; i < item_count; i++)
8419 uint8_t* item = mark_list[i];
8420 mark_list_copy[i] = item;
8422 #endif // defined(_DEBUG) || defined(WRITE_SORT_DATA)
8424 ptrdiff_t start = get_cycle_count();
8426 do_vxsort (mark_list, item_count, low, high);
8428 ptrdiff_t elapsed_cycles = get_cycle_count() - start;
8430 #ifdef WRITE_SORT_DATA
8431 char file_name[256];
8432 sprintf_s (file_name, _countof(file_name), "sort_data_gc%d_heap%d", settings.gc_index, heap_number);
8435 errno_t err = fopen_s (&f, file_name, "wb");
8439 size_t magic = 'SDAT';
8440 if (fwrite (&magic, sizeof(magic), 1, f) != 1)
8441 dprintf (3, ("fwrite failed\n"));
8442 if (fwrite (&elapsed_cycles, sizeof(elapsed_cycles), 1, f) != 1)
8443 dprintf (3, ("fwrite failed\n"));
8444 if (fwrite (&low, sizeof(low), 1, f) != 1)
8445 dprintf (3, ("fwrite failed\n"));
8446 if (fwrite (&item_count, sizeof(item_count), 1, f) != 1)
8447 dprintf (3, ("fwrite failed\n"));
8448 if (fwrite (mark_list_copy, sizeof(mark_list_copy[0]), item_count, f) != item_count)
8449 dprintf (3, ("fwrite failed\n"));
8450 if (fwrite (&magic, sizeof(magic), 1, f) != 1)
8451 dprintf (3, ("fwrite failed\n"));
8452 if (fclose (f) != 0)
8453 dprintf (3, ("fclose failed\n"));
8458 // in debug, sort the copy as well using the proven sort, so we can check we got the right result
8459 if (mark_list_copy_index > mark_list_copy)
8461 introsort::sort (mark_list_copy, mark_list_copy_index - 1, 0);
8463 for (ptrdiff_t i = 0; i < item_count; i++)
8465 uint8_t* item = mark_list[i];
8466 assert (mark_list_copy[i] == item);
8471 dprintf (3, ("Sorting mark lists"));
8472 if (mark_list_index > mark_list)
8474 introsort::sort (mark_list, mark_list_index - 1, 0);
8478 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8479 // start = GetCycleCount32();
8481 // first set the pieces for all heaps to empty
8483 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8485 mark_list_piece_start[heap_num] = NULL;
8486 mark_list_piece_end[heap_num] = NULL;
8489 uint8_t** x = mark_list;
8491 // predicate means: x is still within the mark list, and within the bounds of this heap
8492 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8495 while (x < mark_list_index)
8498 // find the heap x points into - searching cyclically from the last heap,
8499 // because in many cases the right heap is the next one or comes soon after
8501 int last_heap_num = heap_num;
8506 if (heap_num >= n_heaps)
8508 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8509 heap = g_heaps[heap_num];
8511 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8513 // x is the start of the mark list piece for this heap
8514 mark_list_piece_start[heap_num] = x;
8516 // to find the end of the mark list piece for this heap, find the first x
8517 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8520 // let's see if we get lucky and the whole rest belongs to this piece
8521 if (predicate(mark_list_index-1))
8523 x = mark_list_index;
8524 mark_list_piece_end[heap_num] = x;
8528 // we play a variant of binary search to find the point sooner.
8529 // the first loop advances by increasing steps until the predicate turns false.
8530 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8535 uint8_t** temp_x = x;
8542 while (predicate(x));
8543 // we know that only the last step was wrong, so we undo it
8547 // loop invariant - predicate holds at x, but not x + inc
8548 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8550 if (((x + inc) > x) && predicate(x + inc))
8556 // the termination condition and the loop invariant together imply this:
8557 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8558 // so the spot we're looking for is one further
8561 mark_list_piece_end[heap_num] = x;
8566 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8569 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8571 size_t slots_needed = end - start;
8572 size_t slots_available = mark_list_end + 1 - mark_list_index;
8573 size_t slots_to_copy = min(slots_needed, slots_available);
8574 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8575 mark_list_index += slots_to_copy;
8576 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8579 void gc_heap::merge_mark_lists()
8581 uint8_t** source[MAX_SUPPORTED_CPUS];
8582 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8583 int source_heap[MAX_SUPPORTED_CPUS];
8584 int source_count = 0;
8586 // in case of mark list overflow, don't bother
8587 if (mark_list_index > mark_list_end)
8589 // printf("merge_mark_lists: overflow\n");
8593 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8594 // unsigned long start = GetCycleCount32();
8595 for (int i = 0; i < n_heaps; i++)
8597 gc_heap* heap = g_heaps[i];
8598 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8600 source[source_count] = heap->mark_list_piece_start[heap_number];
8601 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8602 source_heap[source_count] = i;
8603 if (source_count < MAX_SUPPORTED_CPUS)
8607 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8609 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8610 #if defined(_DEBUG) || defined(TRACE_GC)
8611 for (int j = 0; j < source_count; j++)
8613 dprintf(3, ("heap_number = %d ", heap_number));
8614 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8615 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8616 // the sources should all be sorted
8617 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8621 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8626 #endif //_DEBUG || TRACE_GC
8628 // start = GetCycleCount32();
8630 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8631 mark_list_index = mark_list;
8632 mark_list_end = &mark_list [mark_list_size-1];
8633 int piece_count = 0;
8634 if (source_count == 0)
8638 else if (source_count == 1)
8640 mark_list = source[0];
8641 mark_list_index = source_end[0];
8642 mark_list_end = mark_list_index;
8647 while (source_count > 1)
8649 // find the lowest and second lowest value in the sources we're merging from
8650 int lowest_source = 0;
8651 uint8_t *lowest = *source[0];
8652 uint8_t *second_lowest = *source[1];
8653 for (int i = 1; i < source_count; i++)
8655 if (lowest > *source[i])
8657 second_lowest = lowest;
8658 lowest = *source[i];
8661 else if (second_lowest > *source[i])
8663 second_lowest = *source[i];
8667 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8669 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8671 if (source_end[lowest_source][-1] <= second_lowest)
8672 x = source_end[lowest_source];
8675 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8676 // but saw no improvement doing that
8677 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8681 // blast this piece to the mark list
8682 append_to_mark_list(source[lowest_source], x);
8685 source[lowest_source] = x;
8687 // check whether this source is now exhausted
8688 if (x >= source_end[lowest_source])
8690 // if it's not the source with the highest index, copy the source with the highest index
8691 // over it so the non-empty sources are always at the beginning
8692 if (lowest_source < source_count-1)
8694 source[lowest_source] = source[source_count-1];
8695 source_end[lowest_source] = source_end[source_count-1];
8700 // we're left with just one source that we copy
8701 append_to_mark_list(source[0], source_end[0]);
8705 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8707 #if defined(_DEBUG) || defined(TRACE_GC)
8708 // the final mark list must be sorted
8709 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8713 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8717 #endif //defined(_DEBUG) || defined(TRACE_GC)
8719 #else //PARALLEL_MARK_LIST_SORT
8720 void gc_heap::combine_mark_lists()
8722 dprintf (3, ("Combining mark lists"));
8723 //verify if a heap has overflowed its mark list
8724 BOOL use_mark_list = TRUE;
8725 for (int i = 0; i < n_heaps; i++)
8727 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8729 use_mark_list = FALSE;
8736 dprintf (3, ("Using mark list"));
8737 //compact the gaps out of the mark list
8739 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8740 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8741 uint8_t** dst_last = current_gap-1;
8743 int srcn = n_heaps-1;
8744 gc_heap* srch = g_heaps [srcn];
8745 uint8_t** src = srch->mark_list_index - 1;
8746 uint8_t** src_beg = srch->mark_list;
8748 while (current_gap <= src)
8750 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8752 //go to the next gap
8754 dprintf (3, ("Going to the next gap %d", gn));
8755 assert (gn < n_heaps);
8756 current_gap = g_heaps [gn]->mark_list_index;
8757 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8758 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8760 while ((srcn > 0) && (src < src_beg))
8762 //go to the previous source
8764 dprintf (3, ("going to the previous source %d", srcn));
8766 gc_heap* srch = g_heaps [srcn];
8767 src = srch->mark_list_index - 1;
8768 src_beg = srch->mark_list;
8770 if (current_gap < src)
8772 dst_last = current_gap;
8773 *current_gap++ = *src--;
8776 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8778 uint8_t** end_of_list = max (src, dst_last);
8780 //sort the resulting compacted list
8781 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8782 if (end_of_list > &g_mark_list[0])
8783 _sort (&g_mark_list[0], end_of_list, 0);
8784 //adjust the mark_list to the beginning of the resulting mark list.
8785 for (int i = 0; i < n_heaps; i++)
8787 g_heaps [i]->mark_list = g_mark_list;
8788 g_heaps [i]->mark_list_index = end_of_list + 1;
8789 g_heaps [i]->mark_list_end = end_of_list + 1;
8794 uint8_t** end_of_list = g_mark_list;
8795 //adjust the mark_list to the beginning of the resulting mark list.
8796 //put the index beyond the end to turn off mark list processing
8797 for (int i = 0; i < n_heaps; i++)
8799 g_heaps [i]->mark_list = g_mark_list;
8800 g_heaps [i]->mark_list_index = end_of_list + 1;
8801 g_heaps [i]->mark_list_end = end_of_list;
8805 #endif // PARALLEL_MARK_LIST_SORT
8806 #endif //MULTIPLE_HEAPS
8808 void gc_heap::grow_mark_list ()
8810 // with vectorized sorting, we can use bigger mark lists
8812 #ifdef MULTIPLE_HEAPS
8813 const size_t MAX_MARK_LIST_SIZE = IsSupportedInstructionSet (InstructionSet::AVX2) ? 1000 * 1024 : 200 * 1024;
8814 #else //MULTIPLE_HEAPS
8815 const size_t MAX_MARK_LIST_SIZE = IsSupportedInstructionSet (InstructionSet::AVX2) ? 32 * 1024 : 16 * 1024;
8816 #endif //MULTIPLE_HEAPS
8818 #ifdef MULTIPLE_HEAPS
8819 const size_t MAX_MARK_LIST_SIZE = 200 * 1024;
8820 #else //MULTIPLE_HEAPS
8821 const size_t MAX_MARK_LIST_SIZE = 16 * 1024;
8822 #endif //MULTIPLE_HEAPS
8825 size_t new_mark_list_size = min (mark_list_size * 2, MAX_MARK_LIST_SIZE);
8826 if (new_mark_list_size == mark_list_size)
8829 #ifdef MULTIPLE_HEAPS
8830 uint8_t** new_mark_list = make_mark_list (new_mark_list_size * n_heaps);
8832 #ifdef PARALLEL_MARK_LIST_SORT
8833 uint8_t** new_mark_list_copy = make_mark_list (new_mark_list_size * n_heaps);
8834 #endif //PARALLEL_MARK_LIST_SORT
8836 if (new_mark_list != nullptr
8837 #ifdef PARALLEL_MARK_LIST_SORT
8838 && new_mark_list_copy != nullptr
8839 #endif //PARALLEL_MARK_LIST_SORT
8842 delete[] g_mark_list;
8843 g_mark_list = new_mark_list;
8844 #ifdef PARALLEL_MARK_LIST_SORT
8845 delete[] g_mark_list_copy;
8846 g_mark_list_copy = new_mark_list_copy;
8847 #endif //PARALLEL_MARK_LIST_SORT
8848 mark_list_size = new_mark_list_size;
8852 delete[] new_mark_list;
8853 #ifdef PARALLEL_MARK_LIST_SORT
8854 delete[] new_mark_list_copy;
8855 #endif //PARALLEL_MARK_LIST_SORT
8858 #else //MULTIPLE_HEAPS
8859 uint8_t** new_mark_list = make_mark_list (new_mark_list_size);
8860 if (new_mark_list != nullptr)
8863 g_mark_list = new_mark_list;
8864 mark_list_size = new_mark_list_size;
8866 #endif //MULTIPLE_HEAPS
8870 class seg_free_spaces
8872 struct seg_free_space
8878 struct free_space_bucket
8880 seg_free_space* free_space;
8881 ptrdiff_t count_add; // Assigned when we first construct the array.
8882 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8885 void move_bucket (int old_power2, int new_power2)
8887 // PREFAST warning 22015: old_power2 could be negative
8888 assert (old_power2 >= 0);
8889 assert (old_power2 >= new_power2);
8891 if (old_power2 == new_power2)
8896 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8897 for (int i = old_power2; i > new_power2; i--)
8899 seg_free_space** dest = &(free_space_buckets[i].free_space);
8902 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8903 if (i > (new_power2 + 1))
8905 seg_free_space temp = *src_index;
8906 *src_index = *dest_index;
8909 src_index = dest_index;
8912 free_space_buckets[old_power2].count_fit--;
8913 free_space_buckets[new_power2].count_fit++;
8918 void dump_free_space (seg_free_space* item)
8925 mark* m = (mark*)(item->start);
8926 len = pinned_len (m);
8927 addr = pinned_plug (m) - len;
8931 heap_segment* seg = (heap_segment*)(item->start);
8932 addr = heap_segment_plan_allocated (seg);
8933 len = heap_segment_committed (seg) - addr;
8936 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8941 seg_free_space* item = NULL;
8944 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8945 for (i = 0; i < (free_space_bucket_count - 1); i++)
8947 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8948 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8949 item = free_space_buckets[i].free_space;
8950 while (item < free_space_buckets[i + 1].free_space)
8952 dump_free_space (item);
8955 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8958 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8959 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8960 item = free_space_buckets[i].free_space;
8962 while (item <= &seg_free_space_array[free_space_item_count - 1])
8964 dump_free_space (item);
8967 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8972 free_space_bucket* free_space_buckets;
8973 seg_free_space* seg_free_space_array;
8974 ptrdiff_t free_space_bucket_count;
8975 ptrdiff_t free_space_item_count;
8979 BOOL has_end_of_seg;
8984 seg_free_spaces (int h_number)
8986 heap_num = h_number;
8991 size_t total_prealloc_size =
8992 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8993 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8995 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8997 return (!!free_space_buckets);
9000 // We take the ordered free space array we got from the 1st pass,
9001 // and feed the portion that we decided to use to this method, ie,
9002 // the largest item_count free spaces.
9003 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
9005 assert (free_space_buckets);
9006 assert (item_count <= (size_t)MAX_PTR);
9008 free_space_bucket_count = bucket_count;
9009 free_space_item_count = item_count;
9012 has_end_of_seg = FALSE;
9015 ptrdiff_t total_item_count = 0;
9018 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
9020 for (i = 0; i < (ptrdiff_t)item_count; i++)
9022 seg_free_space_array[i].start = 0;
9023 seg_free_space_array[i].is_plug = FALSE;
9026 for (i = 0; i < bucket_count; i++)
9028 free_space_buckets[i].count_add = ordered_free_spaces[i];
9029 free_space_buckets[i].count_fit = ordered_free_spaces[i];
9030 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
9031 total_item_count += free_space_buckets[i].count_add;
9034 assert (total_item_count == (ptrdiff_t)item_count);
9037 // If we are adding a free space before a plug we pass the
9038 // mark stack position so we can update the length; we could
9039 // also be adding the free space after the last plug in which
9040 // case start is the segment which we'll need to update the
9041 // heap_segment_plan_allocated.
9042 void add (void* start, BOOL plug_p, BOOL first_p)
9044 size_t size = (plug_p ?
9045 pinned_len ((mark*)start) :
9046 (heap_segment_committed ((heap_segment*)start) -
9047 heap_segment_plan_allocated ((heap_segment*)start)));
9051 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
9055 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
9057 has_end_of_seg = TRUE;
9063 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
9064 size -= eph_gen_starts;
9067 mark* m = (mark*)(start);
9068 pinned_len (m) -= eph_gen_starts;
9072 heap_segment* seg = (heap_segment*)start;
9073 heap_segment_plan_allocated (seg) += eph_gen_starts;
9077 int bucket_power2 = index_of_highest_set_bit (size);
9078 if (bucket_power2 < base_power2)
9083 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
9085 seg_free_space* bucket_free_space = bucket->free_space;
9086 assert (plug_p || (!plug_p && bucket->count_add));
9088 if (bucket->count_add == 0)
9090 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
9094 ptrdiff_t index = bucket->count_add - 1;
9096 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
9099 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
9100 heap_segment_plan_allocated ((heap_segment*)start)),
9106 bucket_free_space[index].is_plug = TRUE;
9109 bucket_free_space[index].start = start;
9110 bucket->count_add--;
9115 // Do a consistency check after all free spaces are added.
9119 int end_of_seg_count = 0;
9121 for (i = 0; i < free_space_item_count; i++)
9123 assert (seg_free_space_array[i].start);
9124 if (!(seg_free_space_array[i].is_plug))
9132 assert (end_of_seg_count == 1);
9136 assert (end_of_seg_count == 0);
9139 for (i = 0; i < free_space_bucket_count; i++)
9141 assert (free_space_buckets[i].count_add == 0);
9147 uint8_t* fit (uint8_t* old_loc,
9149 REQD_ALIGN_AND_OFFSET_DCL)
9154 assert (!is_plug_padded (old_loc));
9155 #endif //SHORT_PLUGS
9156 assert (!node_realigned (old_loc));
9159 size_t saved_plug_size = plug_size;
9161 #ifdef FEATURE_STRUCTALIGN
9162 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
9163 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
9164 #endif // FEATURE_STRUCTALIGN
9166 size_t plug_size_to_fit = plug_size;
9168 // best fit is only done for gen1 to gen2 and we do not pad in gen2.
9169 // however we must account for requirements of large alignment.
9170 // which may result in realignment padding.
9171 #ifdef RESPECT_LARGE_ALIGNMENT
9172 plug_size_to_fit += switch_alignment_size(FALSE);
9173 #endif //RESPECT_LARGE_ALIGNMENT
9175 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
9177 uint8_t* new_address = 0;
9179 if (plug_power2 < base_power2)
9181 plug_power2 = base_power2;
9184 int chosen_power2 = plug_power2 - base_power2;
9186 for (i = chosen_power2; i < free_space_bucket_count; i++)
9188 if (free_space_buckets[i].count_fit != 0)
9195 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
9199 (chosen_power2 + base_power2)));
9201 assert (i < free_space_bucket_count);
9203 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
9204 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
9205 size_t new_free_space_size = 0;
9206 BOOL can_fit = FALSE;
9209 for (i = 0; i < free_space_count; i++)
9211 size_t free_space_size = 0;
9214 if (bucket_free_space[i].is_plug)
9216 mark* m = (mark*)(bucket_free_space[i].start);
9217 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
9219 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start)))
9221 pad = switch_alignment_size (FALSE);
9224 plug_size = saved_plug_size + pad;
9226 free_space_size = pinned_len (m);
9227 new_address = pinned_plug (m) - pinned_len (m);
9229 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
9230 free_space_size == plug_size)
9232 new_free_space_size = free_space_size - plug_size;
9233 pinned_len (m) = new_free_space_size;
9234 #ifdef SIMPLE_DPRINTF
9235 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
9242 index_of_highest_set_bit (free_space_size),
9243 (pinned_plug (m) - pinned_len (m)),
9244 index_of_highest_set_bit (new_free_space_size)));
9245 #endif //SIMPLE_DPRINTF
9249 set_node_realigned (old_loc);
9257 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
9258 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
9260 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
9262 pad = switch_alignment_size (FALSE);
9265 plug_size = saved_plug_size + pad;
9267 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
9268 free_space_size == plug_size)
9270 new_address = heap_segment_plan_allocated (seg);
9271 new_free_space_size = free_space_size - plug_size;
9272 heap_segment_plan_allocated (seg) = new_address + plug_size;
9273 #ifdef SIMPLE_DPRINTF
9274 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
9279 index_of_highest_set_bit (free_space_size),
9280 heap_segment_plan_allocated (seg),
9281 index_of_highest_set_bit (new_free_space_size)));
9282 #endif //SIMPLE_DPRINTF
9285 set_node_realigned (old_loc);
9299 assert (chosen_power2 == 0);
9305 assert ((chosen_power2 && (i == 0)) ||
9306 ((!chosen_power2) && (i < free_space_count)));
9308 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
9310 if (new_bucket_power2 < base_power2)
9312 new_bucket_power2 = base_power2;
9315 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
9324 if (free_space_buckets)
9326 delete [] free_space_buckets;
9328 if (seg_free_space_array)
9330 delete [] seg_free_space_array;
9336 #define marked(i) header(i)->IsMarked()
9337 #define set_marked(i) header(i)->SetMarked()
9338 #define clear_marked(i) header(i)->ClearMarked()
9339 #define pinned(i) header(i)->IsPinned()
9340 #define set_pinned(i) header(i)->SetPinned()
9341 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
9343 inline size_t my_get_size (Object* ob)
9345 MethodTable* mT = header(ob)->GetMethodTable();
9347 return (mT->GetBaseSize() +
9348 (mT->HasComponentSize() ?
9349 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
9352 //#define size(i) header(i)->GetSize()
9353 #define size(i) my_get_size (header(i))
9355 #define contain_pointers(i) header(i)->ContainsPointers()
9356 #ifdef COLLECTIBLE_CLASS
9357 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9359 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9360 #define is_collectible(i) method_table(i)->Collectible()
9361 #else //COLLECTIBLE_CLASS
9362 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9363 #endif //COLLECTIBLE_CLASS
9365 #ifdef BACKGROUND_GC
9367 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9369 uint8_t* range_beg = 0;
9370 uint8_t* range_end = 0;
9371 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9373 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9374 #ifdef FEATURE_BASICFREEZE
9376 #endif // FEATURE_BASICFREEZE
9381 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9383 if ((start < background_saved_highest_address) &&
9384 (end > background_saved_lowest_address))
9386 start = max (start, background_saved_lowest_address);
9387 end = min (end, background_saved_highest_address);
9389 size_t start_mark_bit = mark_bit_of (start);
9390 size_t end_mark_bit = mark_bit_of (end);
9391 unsigned int startbit = mark_bit_bit (start_mark_bit);
9392 unsigned int endbit = mark_bit_bit (end_mark_bit);
9393 size_t startwrd = mark_bit_word (start_mark_bit);
9394 size_t endwrd = mark_bit_word (end_mark_bit);
9396 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9397 (size_t)start, (size_t)start_mark_bit,
9398 (size_t)end, (size_t)end_mark_bit));
9400 unsigned int firstwrd = lowbits (~0, startbit);
9401 unsigned int lastwrd = highbits (~0, endbit);
9403 if (startwrd == endwrd)
9405 unsigned int wrd = firstwrd | lastwrd;
9406 mark_array[startwrd] &= wrd;
9410 // clear the first mark word.
9413 mark_array[startwrd] &= firstwrd;
9417 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9419 mark_array[wrdtmp] = 0;
9422 // clear the last mark word.
9425 mark_array[endwrd] &= lastwrd;
9430 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9432 if ((start < background_saved_highest_address) &&
9433 (end > background_saved_lowest_address))
9435 start = max (start, background_saved_lowest_address);
9436 end = min (end, background_saved_highest_address);
9438 clear_batch_mark_array_bits (start, end);
9441 #endif //BACKGROUND_GC
9444 BOOL gc_heap::is_mark_set (uint8_t* o)
9449 #if defined (_MSC_VER) && defined (TARGET_X86)
9450 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9451 #endif //_MSC_VER && TARGET_X86
9453 // return the generation number of an object.
9454 // It is assumed that the object is valid.
9455 // Note that this will return max_generation for UOH objects
9456 int gc_heap::object_gennum (uint8_t* o)
9458 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9459 (o >= generation_allocation_start (generation_of (max_generation-1))))
9461 // in an ephemeral generation.
9462 for ( int i = 0; i < max_generation-1; i++)
9464 if ((o >= generation_allocation_start (generation_of (i))))
9467 return max_generation-1;
9471 return max_generation;
9475 int gc_heap::object_gennum_plan (uint8_t* o)
9477 if (in_range_for_segment (o, ephemeral_heap_segment))
9479 for (int i = 0; i < ephemeral_generation_count; i++)
9481 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9482 if (plan_start && (o >= plan_start))
9488 return max_generation;
9491 #if defined(_MSC_VER) && defined(TARGET_X86)
9492 #pragma optimize("", on) // Go back to command line default optimizations
9493 #endif //_MSC_VER && TARGET_X86
9495 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, gc_oh_num oh, int h_number)
9497 assert(oh != gc_oh_num::none);
9498 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9500 if (!virtual_commit (new_pages, initial_commit, oh, h_number))
9505 heap_segment* new_segment = (heap_segment*)new_pages;
9507 uint8_t* start = new_pages + segment_info_size;
9508 heap_segment_mem (new_segment) = start;
9509 heap_segment_used (new_segment) = start;
9510 heap_segment_reserved (new_segment) = new_pages + size;
9511 heap_segment_committed (new_segment) = (use_large_pages_p ? heap_segment_reserved(new_segment) : (new_pages + initial_commit));
9512 init_heap_segment (new_segment);
9513 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9517 void gc_heap::init_heap_segment (heap_segment* seg)
9520 heap_segment_next (seg) = 0;
9521 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9522 heap_segment_allocated (seg) = heap_segment_mem (seg);
9523 #ifdef BACKGROUND_GC
9524 heap_segment_background_allocated (seg) = 0;
9525 heap_segment_saved_bg_allocated (seg) = 0;
9526 #endif //BACKGROUND_GC
9529 //Releases the segment to the OS.
9530 // this is always called on one thread only so calling seg_table->remove is fine.
9531 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9533 if (!heap_segment_uoh_p (seg))
9535 //cleanup the brick table back to the empty value
9536 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9539 if (consider_hoarding)
9541 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9542 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9543 //Don't keep the big ones.
9544 if (ss <= INITIAL_ALLOC)
9546 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9547 #ifdef BACKGROUND_GC
9548 // We don't need to clear the decommitted flag because when this segment is used
9549 // for a new segment the flags will be cleared.
9550 if (!heap_segment_decommitted_p (seg))
9551 #endif //BACKGROUND_GC
9553 decommit_heap_segment (seg);
9556 seg_mapping_table_remove_segment (seg);
9558 heap_segment_next (seg) = segment_standby_list;
9559 segment_standby_list = seg;
9566 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9567 heap_number, (size_t)seg,
9568 (size_t)(heap_segment_reserved (seg))));
9570 #ifdef BACKGROUND_GC
9571 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9572 settings.gc_index, current_bgc_state,
9574 decommit_mark_array_by_seg (seg);
9575 #endif //BACKGROUND_GC
9577 seg_mapping_table_remove_segment (seg);
9578 release_segment (seg);
9582 //resets the pages beyond allocates size so they won't be swapped out and back in
9584 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9586 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9587 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9589 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9592 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9595 if (use_large_pages_p)
9597 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9598 size_t size = heap_segment_committed (seg) - page_start;
9599 extra_space = align_on_page (extra_space);
9600 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), MIN_DECOMMIT_SIZE))
9602 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9603 decommit_heap_segment_pages_worker (seg, page_start);
9607 size_t gc_heap::decommit_heap_segment_pages_worker (heap_segment* seg,
9608 uint8_t* new_committed)
9610 assert (!use_large_pages_p);
9611 uint8_t* page_start = align_on_page (new_committed);
9612 size_t size = heap_segment_committed (seg) - page_start;
9615 bool decommit_succeeded_p = virtual_decommit (page_start, size, heap_segment_oh (seg), heap_number);
9616 if (decommit_succeeded_p)
9618 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9620 (size_t)(page_start + size),
9622 heap_segment_committed (seg) = page_start;
9623 if (heap_segment_used (seg) > heap_segment_committed (seg))
9625 heap_segment_used (seg) = heap_segment_committed (seg);
9630 dprintf (3, ("Decommitting heap segment failed"));
9636 //decommit all pages except one or 2
9637 void gc_heap::decommit_heap_segment (heap_segment* seg)
9639 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9641 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9643 #ifdef BACKGROUND_GC
9644 page_start += OS_PAGE_SIZE;
9645 #endif //BACKGROUND_GC
9647 size_t size = heap_segment_committed (seg) - page_start;
9648 bool decommit_succeeded_p = virtual_decommit (page_start, size, heap_segment_oh (seg), heap_number);
9650 if (decommit_succeeded_p)
9652 //re-init the segment object
9653 heap_segment_committed (seg) = page_start;
9654 if (heap_segment_used (seg) > heap_segment_committed (seg))
9656 heap_segment_used (seg) = heap_segment_committed (seg);
9661 void gc_heap::clear_gen0_bricks()
9663 if (!gen0_bricks_cleared)
9665 gen0_bricks_cleared = TRUE;
9666 //initialize brick table for gen 0
9667 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9668 b < brick_of (align_on_brick
9669 (heap_segment_allocated (ephemeral_heap_segment)));
9677 #ifdef BACKGROUND_GC
9678 void gc_heap::rearrange_small_heap_segments()
9680 heap_segment* seg = freeable_soh_segment;
9683 heap_segment* next_seg = heap_segment_next (seg);
9684 // TODO: we need to consider hoarding here.
9685 delete_heap_segment (seg, FALSE);
9688 freeable_soh_segment = 0;
9690 #endif //BACKGROUND_GC
9692 void gc_heap::rearrange_uoh_segments()
9694 dprintf (2, ("deleting empty large segments"));
9695 heap_segment* seg = freeable_uoh_segment;
9698 heap_segment* next_seg = heap_segment_next (seg);
9699 delete_heap_segment (seg, GCConfig::GetRetainVM());
9702 freeable_uoh_segment = 0;
9705 void gc_heap::rearrange_heap_segments(BOOL compacting)
9708 generation_start_segment (generation_of (max_generation));
9710 heap_segment* prev_seg = 0;
9711 heap_segment* next_seg = 0;
9714 next_seg = heap_segment_next (seg);
9716 //link ephemeral segment when expanding
9717 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9719 seg->next = ephemeral_heap_segment;
9720 next_seg = heap_segment_next (seg);
9723 //re-used expanded heap segment
9724 if ((seg == ephemeral_heap_segment) && next_seg)
9726 heap_segment_next (prev_seg) = next_seg;
9727 heap_segment_next (seg) = 0;
9731 uint8_t* end_segment = (compacting ?
9732 heap_segment_plan_allocated (seg) :
9733 heap_segment_allocated (seg));
9734 // check if the segment was reached by allocation
9735 if ((end_segment == heap_segment_mem (seg))&&
9736 !heap_segment_read_only_p (seg))
9738 //if not, unthread and delete
9740 assert (seg != ephemeral_heap_segment);
9741 heap_segment_next (prev_seg) = next_seg;
9742 delete_heap_segment (seg, GCConfig::GetRetainVM());
9744 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9748 if (!heap_segment_read_only_p (seg))
9752 heap_segment_allocated (seg) =
9753 heap_segment_plan_allocated (seg);
9756 // reset the pages between allocated and committed.
9757 if (seg != ephemeral_heap_segment)
9759 decommit_heap_segment_pages (seg, 0);
9772 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9776 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9779 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9781 if (!card_bundle_set_p (x))
9783 assert (!"Card bundle not set");
9784 dprintf (3, ("Card bundle %Ix not set", x));
9788 UNREFERENCED_PARAMETER(first_card_word);
9789 UNREFERENCED_PARAMETER(last_card_word);
9793 // Verifies that any bundles that are not set represent only cards that are not set.
9794 inline void gc_heap::verify_card_bundles()
9797 size_t lowest_card = card_word (card_of (lowest_address));
9798 size_t highest_card = card_word (card_of (highest_address));
9799 size_t cardb = cardw_card_bundle (lowest_card);
9800 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9802 while (cardb < end_cardb)
9804 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9805 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9807 if (card_bundle_set_p (cardb) == 0)
9809 // Verify that no card is set
9810 while (card_word < card_word_end)
9812 if (*card_word != 0)
9814 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9815 dd_collection_count (dynamic_data_of (0)),
9816 (size_t)(card_word-&card_table[0]),
9817 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9820 assert((*card_word)==0);
9830 // If card bundles are enabled, use write watch to find pages in the card table that have
9831 // been dirtied, and set the corresponding card bundle bits.
9832 void gc_heap::update_card_table_bundle()
9834 if (card_bundles_enabled())
9836 // The address of the card word containing the card representing the lowest heap address
9837 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9839 // The address of the card word containing the card representing the highest heap address
9840 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9842 uint8_t* saved_base_address = base_address;
9843 uintptr_t bcount = array_size;
9844 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9848 size_t region_size = align_on_page (high_address) - base_address;
9850 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9851 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9854 (void**)g_addresses,
9856 assert (success && "GetWriteWatch failed!");
9858 dprintf (3,("Found %d pages written", bcount));
9859 for (unsigned i = 0; i < bcount; i++)
9861 // Offset of the dirty page from the start of the card table (clamped to base_address)
9862 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9864 // Offset of the end of the page from the start of the card table (clamped to high addr)
9865 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9866 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9868 // Set the card bundle bits representing the dirty card table page
9869 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9870 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9872 verify_card_bundle_bits_set(bcardw, ecardw);
9875 if (bcount >= array_size)
9877 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9878 bcount = array_size;
9881 } while ((bcount >= array_size) && (base_address < high_address));
9883 // Now that we've updated the card bundle bits, reset the write-tracking state.
9884 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9887 #endif //CARD_BUNDLE
9890 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9892 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9893 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9894 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9895 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9896 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9900 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9902 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9903 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9904 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9905 UNREFERENCED_PARAMETER(is_runtime_suspended);
9906 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9908 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9911 const size_t ww_reset_quantum = 128*1024*1024;
9914 void gc_heap::switch_one_quantum()
9916 enable_preemptive ();
9917 GCToOSInterface::Sleep (1);
9918 disable_preemptive (true);
9921 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9923 size_t reset_size = 0;
9924 size_t remaining_reset_size = 0;
9925 size_t next_reset_size = 0;
9927 while (reset_size != total_reset_size)
9929 remaining_reset_size = total_reset_size - reset_size;
9930 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9931 if (next_reset_size)
9933 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9934 reset_size += next_reset_size;
9936 switch_one_quantum();
9940 assert (reset_size == total_reset_size);
9943 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9944 // we do concurrently.
9945 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9949 *current_total_reset_size += last_reset_size;
9951 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9953 if (*current_total_reset_size > ww_reset_quantum)
9955 switch_one_quantum();
9957 *current_total_reset_size = 0;
9962 void gc_heap::reset_write_watch (BOOL concurrent_p)
9964 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9965 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9966 assert(!concurrent_p);
9967 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9969 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9971 size_t reset_size = 0;
9973 for (int i = max_generation; i < total_generation_count; i++)
9975 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (i)));
9979 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9980 base_address = max (base_address, background_saved_lowest_address);
9982 uint8_t* high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9983 high_address = min (high_address, background_saved_highest_address);
9985 if (base_address < high_address)
9987 size_t reset_size = 0;
9988 size_t region_size = high_address - base_address;
9989 dprintf (3, ("h%d, gen: %Ix, ww: [%Ix(%Id)", heap_number, i, (size_t)base_address, region_size));
9990 //reset_ww_by_chunk (base_address, region_size);
9991 reset_write_watch_for_gc_heap(base_address, region_size);
9992 switch_on_reset (concurrent_p, &reset_size, region_size);
9995 seg = heap_segment_next_rw (seg);
9997 concurrent_print_time_delta (i == max_generation ? "CRWW soh": "CRWW uoh");
10002 #endif //WRITE_WATCH
10004 #ifdef BACKGROUND_GC
10005 void gc_heap::restart_vm()
10007 //assert (generation_allocation_pointer (youngest_generation) == 0);
10008 dprintf (3, ("Restarting EE"));
10009 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Restarting EE\n");
10010 ee_proceed_event.Set();
10014 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
10016 if (awr != awr_ignored)
10020 FIRE_EVENT(BGCAllocWaitBegin, awr);
10024 FIRE_EVENT(BGCAllocWaitEnd, awr);
10030 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
10032 fire_alloc_wait_event (awr, TRUE);
10036 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
10038 fire_alloc_wait_event (awr, FALSE);
10040 #endif //BACKGROUND_GC
10041 void gc_heap::make_generation (int gen_num, heap_segment* seg, uint8_t* start)
10043 generation* gen = generation_of (gen_num);
10045 gen->gen_num = gen_num;
10046 gen->allocation_start = start;
10047 gen->allocation_context.alloc_ptr = 0;
10048 gen->allocation_context.alloc_limit = 0;
10049 gen->allocation_context.alloc_bytes = 0;
10050 gen->allocation_context.alloc_bytes_uoh = 0;
10051 gen->allocation_context_start_region = 0;
10052 gen->start_segment = seg;
10053 gen->allocation_segment = seg;
10054 gen->plan_allocation_start = 0;
10055 gen->free_list_space = 0;
10056 gen->pinned_allocated = 0;
10057 gen->free_list_allocated = 0;
10058 gen->end_seg_allocated = 0;
10059 gen->condemned_allocated = 0;
10060 gen->sweep_allocated = 0;
10061 gen->free_obj_space = 0;
10062 gen->allocation_size = 0;
10063 gen->pinned_allocation_sweep_size = 0;
10064 gen->pinned_allocation_compact_size = 0;
10065 gen->allocate_end_seg_p = FALSE;
10066 gen->free_list_allocator.clear();
10068 #ifdef FREE_USAGE_STATS
10069 memset (gen->gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
10070 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
10071 memset (gen->gen_plugs, 0, sizeof (gen.gen_plugs));
10072 #endif //FREE_USAGE_STATS
10075 void gc_heap::adjust_ephemeral_limits ()
10077 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
10078 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
10080 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
10081 (size_t)ephemeral_low, (size_t)ephemeral_high))
10083 #ifndef MULTIPLE_HEAPS
10084 // This updates the write barrier helpers with the new info.
10085 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
10086 #endif // MULTIPLE_HEAPS
10089 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
10090 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
10094 if (!temp_logfile_name.Get())
10099 char logfile_name[MAX_LONGPATH+1];
10100 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
10101 const char* suffix = is_config ? ".config.log" : ".log";
10102 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
10103 logFile = fopen(logfile_name, "wb");
10106 #endif //TRACE_GC || GC_CONFIG_DRIVEN
10108 size_t gc_heap::get_segment_size_hard_limit (uint32_t* num_heaps, bool should_adjust_num_heaps)
10110 assert (heap_hard_limit);
10111 size_t aligned_hard_limit = align_on_segment_hard_limit (heap_hard_limit);
10112 if (should_adjust_num_heaps)
10114 uint32_t max_num_heaps = (uint32_t)(aligned_hard_limit / min_segment_size_hard_limit);
10115 if (*num_heaps > max_num_heaps)
10117 *num_heaps = max_num_heaps;
10121 size_t seg_size = aligned_hard_limit / *num_heaps;
10122 size_t aligned_seg_size = (use_large_pages_p ? align_on_segment_hard_limit (seg_size) : round_up_power2 (seg_size));
10124 assert (g_theGCHeap->IsValidSegmentSize (aligned_seg_size));
10126 size_t seg_size_from_config = (size_t)GCConfig::GetSegmentSize();
10127 if (seg_size_from_config)
10129 size_t aligned_seg_size_config = (use_large_pages_p ? align_on_segment_hard_limit (seg_size) : round_up_power2 (seg_size_from_config));
10131 aligned_seg_size = max (aligned_seg_size, aligned_seg_size_config);
10134 //printf ("limit: %Idmb, aligned: %Idmb, %d heaps, seg size from config: %Idmb, seg size %Idmb",
10135 // (heap_hard_limit / 1024 / 1024),
10136 // (aligned_hard_limit / 1024 / 1024),
10138 // (seg_size_from_config / 1024 / 1024),
10139 // (aligned_seg_size / 1024 / 1024));
10140 return aligned_seg_size;
10143 HRESULT gc_heap::initialize_gc (size_t soh_segment_size,
10144 size_t loh_segment_size,
10145 size_t poh_segment_size
10146 #ifdef MULTIPLE_HEAPS
10147 ,int number_of_heaps
10148 #endif //MULTIPLE_HEAPS
10152 if (GCConfig::GetLogEnabled())
10154 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
10156 if (gc_log == NULL)
10159 // GCLogFileSize in MBs.
10160 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
10162 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
10168 gc_log_lock.Initialize();
10169 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
10170 if (!gc_log_buffer)
10176 memset (gc_log_buffer, '*', gc_log_buffer_size);
10178 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
10182 #ifdef GC_CONFIG_DRIVEN
10183 if (GCConfig::GetConfigLogEnabled())
10185 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
10187 if (gc_config_log == NULL)
10190 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
10191 if (!gc_config_log_buffer)
10193 fclose(gc_config_log);
10197 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
10199 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
10200 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
10201 "h#", // heap index
10204 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
10205 "EX", // heap expansion
10206 "NF", // normal fit
10207 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
10210 "PreS", // short object before pinned plug
10211 "PostS", // short object after pinned plug
10212 "Merge", // merged pinned plugs
10213 "Conv", // converted to pinned plug
10214 "Pre", // plug before pinned plug but not after
10215 "Post", // plug after pinned plug but not before
10216 "PrPo", // plug both before and after pinned plug
10217 "PreP", // pre short object padded
10218 "PostP" // post short object padded
10221 #endif //GC_CONFIG_DRIVEN
10223 HRESULT hres = S_OK;
10226 hardware_write_watch_api_supported();
10227 #ifdef BACKGROUND_GC
10228 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
10230 gc_can_use_concurrent = true;
10231 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10232 virtual_alloc_hardware_write_watch = true;
10233 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10237 gc_can_use_concurrent = false;
10239 #endif //BACKGROUND_GC
10240 #endif //WRITE_WATCH
10242 #ifdef BACKGROUND_GC
10243 // leave the first page to contain only segment info
10244 // because otherwise we could need to revisit the first page frequently in
10246 segment_info_size = OS_PAGE_SIZE;
10248 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
10249 #endif //BACKGROUND_GC
10251 reserved_memory = 0;
10252 size_t initial_heap_size = soh_segment_size + loh_segment_size + poh_segment_size;
10253 #ifdef MULTIPLE_HEAPS
10254 reserved_memory_limit = initial_heap_size * number_of_heaps;
10255 #else //MULTIPLE_HEAPS
10256 reserved_memory_limit = initial_heap_size;
10257 int number_of_heaps = 1;
10258 #endif //MULTIPLE_HEAPS
10260 if (heap_hard_limit)
10262 check_commit_cs.Initialize();
10265 bool separated_poh_p = use_large_pages_p && heap_hard_limit_oh[0] && (GCConfig::GetGCHeapHardLimitPOH() == 0) && (GCConfig::GetGCHeapHardLimitPOHPercent() == 0);
10266 if (!reserve_initial_memory (soh_segment_size, loh_segment_size, poh_segment_size, number_of_heaps, use_large_pages_p, separated_poh_p))
10267 return E_OUTOFMEMORY;
10270 //check if we need to turn on card_bundles.
10271 #ifdef MULTIPLE_HEAPS
10272 // use INT64 arithmetic here because of possible overflow on 32p
10273 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
10275 // use INT64 arithmetic here because of possible overflow on 32p
10276 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
10277 #endif //MULTIPLE_HEAPS
10279 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
10281 settings.card_bundles = TRUE;
10285 settings.card_bundles = FALSE;
10287 #endif //CARD_BUNDLE
10289 settings.first_init();
10291 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
10292 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
10294 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
10297 init_static_data();
10299 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
10301 if (!g_gc_card_table)
10302 return E_OUTOFMEMORY;
10304 gc_started = FALSE;
10306 #ifdef MULTIPLE_HEAPS
10307 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
10309 return E_OUTOFMEMORY;
10312 #pragma warning(push)
10313 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10314 #endif // _PREFAST_
10315 g_promoted = new (nothrow) size_t [number_of_heaps*16];
10316 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
10318 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10319 #endif //MH_SC_MARK
10321 #pragma warning(pop)
10322 #endif // _PREFAST_
10323 if (!g_promoted || !g_bpromoted)
10324 return E_OUTOFMEMORY;
10327 if (!g_mark_stack_busy)
10328 return E_OUTOFMEMORY;
10329 #endif //MH_SC_MARK
10331 if (!create_thread_support (number_of_heaps))
10332 return E_OUTOFMEMORY;
10334 if (!heap_select::init (number_of_heaps))
10335 return E_OUTOFMEMORY;
10337 #endif //MULTIPLE_HEAPS
10339 #ifdef MULTIPLE_HEAPS
10340 yp_spin_count_unit = 32 * number_of_heaps;
10342 yp_spin_count_unit = 32 * g_num_processors;
10343 #endif //MULTIPLE_HEAPS
10345 #if defined(__linux__)
10346 GCToEEInterface::UpdateGCEventStatus(static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Default)),
10347 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Default)),
10348 static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Private)),
10349 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Private)));
10350 #endif // __linux__
10353 InitSupportedInstructionSet ((int32_t)GCConfig::GetGCEnabledInstructionSets());
10356 if (!init_semi_shared())
10364 //Initializes PER_HEAP_ISOLATED data members.
10366 gc_heap::init_semi_shared()
10370 #ifdef BGC_SERVO_TUNING
10371 uint32_t current_memory_load = 0;
10372 uint32_t sweep_flr_goal = 0;
10373 uint32_t sweep_flr_goal_loh = 0;
10374 #endif //BGC_SERVO_TUNING
10376 // This is used for heap expansion - it's to fix exactly the start for gen 0
10377 // through (max_generation-1). When we expand the heap we allocate all these
10378 // gen starts at the beginning of the new ephemeral seg.
10379 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10382 #ifdef MULTIPLE_HEAPS
10383 mark_list_size = min (100*1024, max (8192, soh_segment_size/(2*10*32)));
10384 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10386 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10387 #ifdef PARALLEL_MARK_LIST_SORT
10388 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10389 if (!g_mark_list_copy)
10393 #endif //PARALLEL_MARK_LIST_SORT
10395 #else //MULTIPLE_HEAPS
10397 mark_list_size = max (8192, soh_segment_size/(64*32));
10398 g_mark_list = make_mark_list (mark_list_size);
10400 #endif //MULTIPLE_HEAPS
10402 dprintf (3, ("mark_list_size: %d", mark_list_size));
10410 #ifdef MULTIPLE_HEAPS
10411 // gradual decommit: set size to some reasonable value per time interval
10412 max_decommit_step_size = ((DECOMMIT_SIZE_PER_MILLISECOND * DECOMMIT_TIME_STEP_MILLISECONDS) / n_heaps);
10414 // but do at least MIN_DECOMMIT_SIZE per step to make the OS call worthwhile
10415 max_decommit_step_size = max (max_decommit_step_size, MIN_DECOMMIT_SIZE);
10416 #endif //MULTIPLE_HEAPS
10418 #ifdef FEATURE_BASICFREEZE
10419 seg_table = sorted_table::make_sorted_table();
10423 #endif //FEATURE_BASICFREEZE
10425 segment_standby_list = 0;
10427 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10431 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10436 fgn_loh_percent = 0;
10437 full_gc_approach_event_set = false;
10439 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10441 memset (&last_ephemeral_gc_info, 0, sizeof (last_ephemeral_gc_info));
10442 memset (&last_full_blocking_gc_info, 0, sizeof (last_full_blocking_gc_info));
10443 #ifdef BACKGROUND_GC
10444 memset (&last_bgc_info, 0, sizeof (last_bgc_info));
10445 #endif //BACKGROUND_GC
10447 should_expand_in_full_gc = FALSE;
10449 #ifdef FEATURE_LOH_COMPACTION
10450 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10451 loh_compaction_mode = loh_compaction_default;
10452 #endif //FEATURE_LOH_COMPACTION
10454 loh_size_threshold = (size_t)GCConfig::GetLOHThreshold();
10455 assert (loh_size_threshold >= LARGE_OBJECT_SIZE);
10457 #ifdef BGC_SERVO_TUNING
10458 memset (bgc_tuning::gen_calc, 0, sizeof (bgc_tuning::gen_calc));
10459 memset (bgc_tuning::gen_stats, 0, sizeof (bgc_tuning::gen_stats));
10460 memset (bgc_tuning::current_bgc_end_data, 0, sizeof (bgc_tuning::current_bgc_end_data));
10462 // for the outer loop - the ML (memory load) loop
10463 bgc_tuning::enable_fl_tuning = (GCConfig::GetBGCFLTuningEnabled() != 0);
10464 bgc_tuning::memory_load_goal = (uint32_t)GCConfig::GetBGCMemGoal();
10465 bgc_tuning::memory_load_goal_slack = (uint32_t)GCConfig::GetBGCMemGoalSlack();
10466 bgc_tuning::ml_kp = (double)GCConfig::GetBGCMLkp() / 1000.0;
10467 bgc_tuning::ml_ki = (double)GCConfig::GetBGCMLki() / 1000.0;
10468 bgc_tuning::ratio_correction_step = (double)GCConfig::GetBGCG2RatioStep() / 100.0;
10470 // for the inner loop - the alloc loop which calculates the allocated bytes in gen2 before
10471 // triggering the next BGC.
10472 bgc_tuning::above_goal_kp = (double)GCConfig::GetBGCFLkp() / 1000000.0;
10473 bgc_tuning::enable_ki = (GCConfig::GetBGCFLEnableKi() != 0);
10474 bgc_tuning::above_goal_ki = (double)GCConfig::GetBGCFLki() / 1000000.0;
10475 bgc_tuning::enable_kd = (GCConfig::GetBGCFLEnableKd() != 0);
10476 bgc_tuning::above_goal_kd = (double)GCConfig::GetBGCFLkd() / 100.0;
10477 bgc_tuning::enable_smooth = (GCConfig::GetBGCFLEnableSmooth() != 0);
10478 bgc_tuning::num_gen1s_smooth_factor = (double)GCConfig::GetBGCFLSmoothFactor() / 100.0;
10479 bgc_tuning::enable_tbh = (GCConfig::GetBGCFLEnableTBH() != 0);
10480 bgc_tuning::enable_ff = (GCConfig::GetBGCFLEnableFF() != 0);
10481 bgc_tuning::above_goal_ff = (double)GCConfig::GetBGCFLff() / 100.0;
10482 bgc_tuning::enable_gradual_d = (GCConfig::GetBGCFLGradualD() != 0);
10483 sweep_flr_goal = (uint32_t)GCConfig::GetBGCFLSweepGoal();
10484 sweep_flr_goal_loh = (uint32_t)GCConfig::GetBGCFLSweepGoalLOH();
10486 bgc_tuning::gen_calc[0].sweep_flr_goal = ((sweep_flr_goal == 0) ? 20.0 : (double)sweep_flr_goal);
10487 bgc_tuning::gen_calc[1].sweep_flr_goal = ((sweep_flr_goal_loh == 0) ? 20.0 : (double)sweep_flr_goal_loh);
10489 bgc_tuning::available_memory_goal = (uint64_t)((double)gc_heap::total_physical_mem * (double)(100 - bgc_tuning::memory_load_goal) / 100);
10490 get_memory_info (¤t_memory_load);
10492 dprintf (BGC_TUNING_LOG, ("BTL tuning %s!!!",
10493 (bgc_tuning::enable_fl_tuning ? "enabled" : "disabled")));
10495 #ifdef SIMPLE_DPRINTF
10496 dprintf (BGC_TUNING_LOG, ("BTL tuning parameters: mem goal: %d%%(%I64d), +/-%d%%, gen2 correction factor: %.2f, sweep flr goal: %d%%, smooth factor: %.3f(%s), TBH: %s, FF: %.3f(%s), ml: kp %.5f, ki %.10f",
10497 bgc_tuning::memory_load_goal,
10498 bgc_tuning::available_memory_goal,
10499 bgc_tuning::memory_load_goal_slack,
10500 bgc_tuning::ratio_correction_step,
10501 (int)bgc_tuning::gen_calc[0].sweep_flr_goal,
10502 bgc_tuning::num_gen1s_smooth_factor,
10503 (bgc_tuning::enable_smooth ? "enabled" : "disabled"),
10504 (bgc_tuning::enable_tbh ? "enabled" : "disabled"),
10505 bgc_tuning::above_goal_ff,
10506 (bgc_tuning::enable_ff ? "enabled" : "disabled"),
10508 bgc_tuning::ml_ki));
10510 dprintf (BGC_TUNING_LOG, ("BTL tuning parameters: kp: %.5f, ki: %.5f (%s), kd: %.3f (kd-%s, gd-%s), ff: %.3f",
10511 bgc_tuning::above_goal_kp,
10512 bgc_tuning::above_goal_ki,
10513 (bgc_tuning::enable_ki ? "enabled" : "disabled"),
10514 bgc_tuning::above_goal_kd,
10515 (bgc_tuning::enable_kd ? "enabled" : "disabled"),
10516 (bgc_tuning::enable_gradual_d ? "enabled" : "disabled"),
10517 bgc_tuning::above_goal_ff));
10518 #endif //SIMPLE_DPRINTF
10520 if (bgc_tuning::enable_fl_tuning && (current_memory_load < bgc_tuning::memory_load_goal))
10522 uint32_t distance_to_goal = bgc_tuning::memory_load_goal - current_memory_load;
10523 bgc_tuning::stepping_interval = max (distance_to_goal / 10, 1);
10524 bgc_tuning::last_stepping_mem_load = current_memory_load;
10525 bgc_tuning::last_stepping_bgc_count = 0;
10526 dprintf (BGC_TUNING_LOG, ("current ml: %d, %d to goal, interval: %d",
10527 current_memory_load, distance_to_goal, bgc_tuning::stepping_interval));
10531 dprintf (BGC_TUNING_LOG, ("current ml: %d, >= goal: %d, disable stepping",
10532 current_memory_load, bgc_tuning::memory_load_goal));
10533 bgc_tuning::use_stepping_trigger_p = false;
10535 #endif //BGC_SERVO_TUNING
10537 #ifdef BACKGROUND_GC
10538 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10539 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10540 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10543 int number_bgc_threads = 1;
10544 #ifdef MULTIPLE_HEAPS
10545 number_bgc_threads = n_heaps;
10546 #endif //MULTIPLE_HEAPS
10547 if (!create_bgc_threads_support (number_bgc_threads))
10552 #endif //BACKGROUND_GC
10554 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10556 #ifdef GC_CONFIG_DRIVEN
10557 compact_or_sweep_gcs[0] = 0;
10558 compact_or_sweep_gcs[1] = 0;
10559 #endif //GC_CONFIG_DRIVEN
10562 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10563 #endif //SHORT_PLUGS
10571 if (full_gc_approach_event.IsValid())
10573 full_gc_approach_event.CloseEvent();
10575 if (full_gc_end_event.IsValid())
10577 full_gc_end_event.CloseEvent();
10584 gc_heap* gc_heap::make_gc_heap (
10585 #ifdef MULTIPLE_HEAPS
10588 #endif //MULTIPLE_HEAPS
10593 #ifdef MULTIPLE_HEAPS
10594 res = new (nothrow) gc_heap;
10598 res->vm_heap = vm_hp;
10599 res->alloc_context_count = 0;
10602 #ifdef PARALLEL_MARK_LIST_SORT
10603 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10604 if (!res->mark_list_piece_start)
10608 #pragma warning(push)
10609 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10610 #endif // _PREFAST_
10611 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10613 #pragma warning(pop)
10614 #endif // _PREFAST_
10616 if (!res->mark_list_piece_end)
10618 #endif //PARALLEL_MARK_LIST_SORT
10622 #endif //MULTIPLE_HEAPS
10624 if (res->init_gc_heap (
10625 #ifdef MULTIPLE_HEAPS
10627 #else //MULTIPLE_HEAPS
10629 #endif //MULTIPLE_HEAPS
10635 #ifdef MULTIPLE_HEAPS
10638 return (gc_heap*)1;
10639 #endif //MULTIPLE_HEAPS
10643 gc_heap::wait_for_gc_done(int32_t timeOut)
10645 bool cooperative_mode = enable_preemptive ();
10647 uint32_t dwWaitResult = NOERROR;
10649 gc_heap* wait_heap = NULL;
10650 while (gc_heap::gc_started)
10652 #ifdef MULTIPLE_HEAPS
10653 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL))->pGenGCHeap;
10654 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10655 #endif // MULTIPLE_HEAPS
10658 PREFIX_ASSUME(wait_heap != NULL);
10659 #endif // _PREFAST_
10661 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10663 disable_preemptive (cooperative_mode);
10665 return dwWaitResult;
10669 gc_heap::set_gc_done()
10671 enter_gc_done_event_lock();
10672 if (!gc_done_event_set)
10674 gc_done_event_set = true;
10675 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10676 gc_done_event.Set();
10678 exit_gc_done_event_lock();
10682 gc_heap::reset_gc_done()
10684 enter_gc_done_event_lock();
10685 if (gc_done_event_set)
10687 gc_done_event_set = false;
10688 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10689 gc_done_event.Reset();
10691 exit_gc_done_event_lock();
10695 gc_heap::enter_gc_done_event_lock()
10697 uint32_t dwSwitchCount = 0;
10700 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10702 while (gc_done_event_lock >= 0)
10704 if (g_num_processors > 1)
10706 int spin_count = yp_spin_count_unit;
10707 for (int j = 0; j < spin_count; j++)
10709 if (gc_done_event_lock < 0)
10711 YieldProcessor(); // indicate to the processor that we are spinning
10713 if (gc_done_event_lock >= 0)
10714 GCToOSInterface::YieldThread(++dwSwitchCount);
10717 GCToOSInterface::YieldThread(++dwSwitchCount);
10724 gc_heap::exit_gc_done_event_lock()
10726 gc_done_event_lock = -1;
10729 #ifndef MULTIPLE_HEAPS
10731 #ifdef RECORD_LOH_STATE
10732 int gc_heap::loh_state_index = 0;
10733 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10734 #endif //RECORD_LOH_STATE
10736 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10737 VOLATILE(bool) gc_heap::gc_done_event_set;
10738 GCEvent gc_heap::gc_done_event;
10739 #endif //!MULTIPLE_HEAPS
10740 VOLATILE(bool) gc_heap::internal_gc_done;
10742 void gc_heap::add_saved_spinlock_info (
10744 msl_enter_state enter_state,
10745 msl_take_state take_state)
10748 #ifdef SPINLOCK_HISTORY
10749 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10751 current->enter_state = enter_state;
10752 current->take_state = take_state;
10753 current->thread_id.SetToCurrentThread();
10754 current->loh_p = loh_p;
10755 dprintf (SPINLOCK_LOG, ("[%d]%s %s %s",
10757 (loh_p ? "loh" : "soh"),
10758 ((enter_state == me_acquire) ? "E" : "L"),
10759 msl_take_state_str[take_state]));
10761 spinlock_info_index++;
10763 assert (spinlock_info_index <= max_saved_spinlock_info);
10765 if (spinlock_info_index >= max_saved_spinlock_info)
10767 spinlock_info_index = 0;
10770 UNREFERENCED_PARAMETER(enter_state);
10771 UNREFERENCED_PARAMETER(take_state);
10772 #endif //SPINLOCK_HISTORY
10776 gc_heap::init_gc_heap (int h_number)
10778 #ifdef MULTIPLE_HEAPS
10782 allocated_since_last_gc = 0;
10784 #ifdef SPINLOCK_HISTORY
10785 spinlock_info_index = 0;
10786 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10787 #endif //SPINLOCK_HISTORY
10789 // initialize per heap members.
10790 ephemeral_low = (uint8_t*)1;
10792 ephemeral_high = MAX_PTR;
10794 ephemeral_heap_segment = 0;
10796 oomhist_index_per_heap = 0;
10798 freeable_uoh_segment = 0;
10800 condemned_generation_num = 0;
10802 blocking_collection = FALSE;
10804 generation_skip_ratio = 100;
10806 mark_stack_tos = 0;
10808 mark_stack_bos = 0;
10810 mark_stack_array_length = 0;
10812 mark_stack_array = 0;
10814 #if defined (_DEBUG) && defined (VERIFY_HEAP)
10815 verify_pinned_queue_p = FALSE;
10816 #endif // _DEBUG && VERIFY_HEAP
10818 loh_pinned_queue_tos = 0;
10820 loh_pinned_queue_bos = 0;
10822 loh_pinned_queue_length = 0;
10824 loh_pinned_queue_decay = LOH_PIN_DECAY;
10826 loh_pinned_queue = 0;
10828 min_overflow_address = MAX_PTR;
10830 max_overflow_address = 0;
10832 gen0_bricks_cleared = FALSE;
10834 gen0_must_clear_bricks = 0;
10836 allocation_quantum = CLR_SIZE;
10838 more_space_lock_soh = gc_lock;
10840 more_space_lock_uoh = gc_lock;
10842 ro_segments_in_range = FALSE;
10844 loh_alloc_since_cg = 0;
10846 new_heap_segment = NULL;
10848 gen0_allocated_after_gc_p = false;
10850 #ifdef RECORD_LOH_STATE
10851 loh_state_index = 0;
10852 #endif //RECORD_LOH_STATE
10853 #endif //MULTIPLE_HEAPS
10855 #ifdef MULTIPLE_HEAPS
10856 if (h_number > n_heaps)
10858 assert (!"Number of heaps exceeded");
10862 heap_number = h_number;
10863 #endif //MULTIPLE_HEAPS
10865 memset (&oom_info, 0, sizeof (oom_info));
10866 memset (&fgm_result, 0, sizeof (fgm_result));
10867 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10871 gc_done_event_lock = -1;
10872 gc_done_event_set = false;
10874 heap_segment* seg = make_initial_segment (soh_gen0, h_number);
10878 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10879 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10880 gc_etw_segment_small_object_heap);
10882 seg_mapping_table_add_segment (seg, __this);
10884 #ifdef MULTIPLE_HEAPS
10885 heap_segment_heap (seg) = this;
10886 #endif //MULTIPLE_HEAPS
10888 /* todo: Need a global lock for this */
10889 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10890 own_card_table (ct);
10891 card_table = translate_card_table (ct);
10892 /* End of global lock */
10894 brick_table = card_table_brick_table (ct);
10895 highest_address = card_table_highest_address (ct);
10896 lowest_address = card_table_lowest_address (ct);
10899 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10900 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10901 card_table_card_bundle_table (ct));
10902 #endif //CARD_BUNDLE
10904 #ifdef BACKGROUND_GC
10905 if (gc_can_use_concurrent)
10906 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10909 #endif //BACKGROUND_GC
10911 uint8_t* start = heap_segment_mem (seg);
10913 for (int i = max_generation; i >= 0; i--)
10915 make_generation (i, seg, start);
10916 start += Align (min_obj_size);
10919 heap_segment_allocated (seg) = start;
10920 alloc_allocated = start;
10921 heap_segment_used (seg) = start - plug_skew;
10923 ephemeral_heap_segment = seg;
10925 // Create segments for the large and pinned generations
10926 heap_segment* lseg = make_initial_segment(loh_generation, h_number);
10930 lseg->flags |= heap_segment_flags_loh;
10932 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10933 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10934 gc_etw_segment_large_object_heap);
10936 heap_segment* pseg = make_initial_segment(poh_generation, h_number);
10940 pseg->flags |= heap_segment_flags_poh;
10942 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(pseg),
10943 (size_t)(heap_segment_reserved (pseg) - heap_segment_mem(pseg)),
10944 gc_etw_segment_pinned_object_heap);
10946 seg_mapping_table_add_segment (lseg, __this);
10947 seg_mapping_table_add_segment (pseg, __this);
10949 make_generation (loh_generation, lseg, heap_segment_mem (lseg));
10950 make_generation (poh_generation, pseg, heap_segment_mem (pseg));
10952 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10953 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10955 heap_segment_allocated (pseg) = heap_segment_mem (pseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10956 heap_segment_used (pseg) = heap_segment_allocated (pseg) - plug_skew;
10958 generation_of (max_generation)->free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST_BITS, gen2_alloc_list);
10959 generation_of (loh_generation)->free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST_BITS, loh_alloc_list);
10960 generation_of (poh_generation)->free_list_allocator = allocator(NUM_POH_ALIST, BASE_POH_ALIST_BITS, poh_alloc_list);
10962 for (int gen_num = 0; gen_num < total_generation_count; gen_num++)
10964 generation* gen = generation_of (gen_num);
10965 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10968 #ifdef MULTIPLE_HEAPS
10969 heap_segment_heap (lseg) = this;
10970 heap_segment_heap (pseg) = this;
10972 //initialize the alloc context heap
10973 generation_alloc_context (generation_of (soh_gen0))->set_alloc_heap(vm_heap);
10974 generation_alloc_context (generation_of (loh_generation))->set_alloc_heap(vm_heap);
10975 generation_alloc_context (generation_of (poh_generation))->set_alloc_heap(vm_heap);
10977 #endif //MULTIPLE_HEAPS
10979 if (!init_dynamic_data())
10984 etw_allocation_running_amount[0] = 0;
10985 etw_allocation_running_amount[1] = 0;
10986 total_alloc_bytes_soh = 0;
10987 total_alloc_bytes_uoh = 0;
10989 //needs to be done after the dynamic data has been initialized
10990 #ifndef MULTIPLE_HEAPS
10991 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10992 #endif //!MULTIPLE_HEAPS
10994 fgn_maxgen_percent = 0;
10995 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10997 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
11001 make_mark_stack(arr);
11003 #ifdef BACKGROUND_GC
11004 #ifdef BGC_SERVO_TUNING
11006 loh_a_bgc_marking = 0;
11007 loh_a_bgc_planning = 0;
11008 bgc_maxgen_end_fl_size = 0;
11009 #endif //BGC_SERVO_TUNING
11010 freeable_soh_segment = 0;
11011 gchist_index_per_heap = 0;
11012 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
11016 make_background_mark_stack (b_arr);
11017 #endif //BACKGROUND_GC
11019 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
11020 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
11021 if (heap_number == 0)
11023 stomp_write_barrier_initialize(
11024 #ifdef MULTIPLE_HEAPS
11025 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
11027 ephemeral_low, ephemeral_high
11028 #endif //!MULTIPLE_HEAPS
11032 #ifdef MULTIPLE_HEAPS
11033 get_proc_and_numa_for_heap (heap_number);
11034 if (!create_gc_thread ())
11037 g_heaps [heap_number] = this;
11039 #endif //MULTIPLE_HEAPS
11041 #ifdef FEATURE_PREMORTEM_FINALIZATION
11042 HRESULT hr = AllocateCFinalize(&finalize_queue);
11045 #endif // FEATURE_PREMORTEM_FINALIZATION
11047 max_free_space_items = MAX_NUM_FREE_SPACES;
11049 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
11056 if (!bestfit_seg->alloc())
11061 last_gc_before_oom = FALSE;
11063 sufficient_gen0_space_p = FALSE;
11065 #ifdef MULTIPLE_HEAPS
11067 #ifdef HEAP_ANALYZE
11069 heap_analyze_success = TRUE;
11071 internal_root_array = 0;
11073 internal_root_array_index = 0;
11075 internal_root_array_length = initial_internal_roots;
11079 current_obj_size = 0;
11081 #endif //HEAP_ANALYZE
11083 #endif // MULTIPLE_HEAPS
11085 #ifdef BACKGROUND_GC
11086 bgc_thread_id.Clear();
11088 if (!create_bgc_thread_support())
11093 bgc_alloc_lock = new (nothrow) exclusive_sync;
11094 if (!bgc_alloc_lock)
11099 bgc_alloc_lock->init();
11100 bgc_thread_running = 0;
11102 bgc_threads_timeout_cs.Initialize();
11103 expanded_in_fgc = 0;
11104 current_bgc_state = bgc_not_in_process;
11105 background_soh_alloc_count = 0;
11106 background_uoh_alloc_count = 0;
11107 bgc_overflow_count = 0;
11108 end_loh_size = dd_min_size (dynamic_data_of (loh_generation));
11109 end_poh_size = dd_min_size (dynamic_data_of (poh_generation));
11110 #endif //BACKGROUND_GC
11112 #ifdef GC_CONFIG_DRIVEN
11113 memset(interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
11114 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
11115 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
11116 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
11117 #endif //GC_CONFIG_DRIVEN
11123 gc_heap::destroy_semi_shared()
11125 //TODO: will need to move this to per heap
11126 //#ifdef BACKGROUND_GC
11127 // if (c_mark_list)
11128 // delete c_mark_list;
11129 //#endif //BACKGROUND_GC
11133 delete g_mark_list;
11136 if (seg_mapping_table)
11137 delete seg_mapping_table;
11139 #ifdef FEATURE_BASICFREEZE
11140 //destroy the segment map
11141 seg_table->delete_sorted_table();
11142 #endif //FEATURE_BASICFREEZE
11146 gc_heap::self_destroy()
11148 #ifdef BACKGROUND_GC
11150 #endif //BACKGROUND_GC
11152 if (gc_done_event.IsValid())
11154 gc_done_event.CloseEvent();
11157 // destroy every segment
11158 for (int i = max_generation; i < total_generation_count; i++)
11160 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (i)));
11161 PREFIX_ASSUME(seg != NULL);
11165 heap_segment* next_seg = heap_segment_next_rw (seg);
11166 delete_heap_segment (seg);
11171 // get rid of the card table
11172 release_card_table (card_table);
11174 // destroy the mark stack
11175 delete mark_stack_array;
11177 #ifdef FEATURE_PREMORTEM_FINALIZATION
11178 if (finalize_queue)
11179 delete finalize_queue;
11180 #endif // FEATURE_PREMORTEM_FINALIZATION
11184 gc_heap::destroy_gc_heap(gc_heap* heap)
11186 heap->self_destroy();
11190 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
11191 // the finalizer queue has been drained.
11192 void gc_heap::shutdown_gc()
11194 destroy_semi_shared();
11196 #ifdef MULTIPLE_HEAPS
11197 //delete the heaps array
11199 destroy_thread_support();
11201 #endif //MULTIPLE_HEAPS
11202 //destroy seg_manager
11204 destroy_initial_memory();
11206 GCToOSInterface::Shutdown();
11210 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11211 uint8_t* old_loc, int use_padding)
11213 BOOL already_padded = FALSE;
11215 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
11217 alloc_pointer = alloc_pointer + Align (min_obj_size);
11218 already_padded = TRUE;
11220 #endif //SHORT_PLUGS
11222 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
11223 size = size + switch_alignment_size (already_padded);
11225 #ifdef FEATURE_STRUCTALIGN
11226 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
11227 #endif // FEATURE_STRUCTALIGN
11229 // in allocate_in_condemned_generation we can have this when we
11230 // set the alloc_limit to plan_allocated which could be less than
11232 if (alloc_limit < alloc_pointer)
11239 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
11241 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
11242 #else //SHORT_PLUGS
11243 ||((alloc_pointer + size) == alloc_limit)
11244 #endif //SHORT_PLUGS
11249 assert (size == Align (min_obj_size));
11250 return ((size_t)(alloc_limit - alloc_pointer) >= size);
11255 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11258 // We could have run into cases where this is true when alloc_allocated is the
11259 // the same as the seg committed.
11260 if (alloc_limit < alloc_pointer)
11265 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
11268 // Grow by committing more pages
11269 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* hard_limit_exceeded_p)
11271 assert (high_address <= heap_segment_reserved (seg));
11273 if (hard_limit_exceeded_p)
11274 *hard_limit_exceeded_p = false;
11276 //return 0 if we are at the end of the segment.
11277 if (align_on_page (high_address) > heap_segment_reserved (seg))
11280 if (high_address <= heap_segment_committed (seg))
11283 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
11284 c_size = max (c_size, commit_min_th);
11285 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
11290 STRESS_LOG2(LF_GC, LL_INFO10000,
11291 "Growing heap_segment: %Ix high address: %Ix\n",
11292 (size_t)seg, (size_t)high_address);
11294 bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_segment_oh (seg), heap_number, hard_limit_exceeded_p);
11297 heap_segment_committed (seg) += c_size;
11299 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix\n",
11300 (size_t)heap_segment_committed (seg));
11302 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
11303 assert (high_address <= heap_segment_committed (seg));
11310 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
11312 BOOL already_padded = FALSE;
11314 if ((old_loc != 0) && pad_front_p)
11316 allocated = allocated + Align (min_obj_size);
11317 already_padded = TRUE;
11319 #endif //SHORT_PLUGS
11321 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
11322 size += switch_alignment_size (already_padded);
11324 #ifdef FEATURE_STRUCTALIGN
11325 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
11326 return grow_heap_segment (seg, allocated + pad + size);
11327 #else // FEATURE_STRUCTALIGN
11328 return grow_heap_segment (seg, allocated + size);
11329 #endif // FEATURE_STRUCTALIGN
11332 //used only in older generation allocation (i.e during gc).
11333 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen)
11335 dprintf (3, ("gc Expanding segment allocation"));
11336 heap_segment* seg = generation_allocation_segment (gen);
11337 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
11339 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
11341 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
11342 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
11343 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
11347 uint8_t* hole = generation_allocation_pointer (gen);
11348 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
11352 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
11353 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
11354 if (size >= Align (min_free_list))
11356 if (allocated_size < min_free_list)
11358 if (size >= (Align (min_free_list) + Align (min_obj_size)))
11360 //split hole into min obj + threadable free item
11361 make_unused_array (hole, min_obj_size);
11362 generation_free_obj_space (gen) += Align (min_obj_size);
11363 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
11364 generation_free_list_space (gen) += size - Align (min_obj_size);
11365 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
11366 size - Align (min_obj_size));
11367 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
11371 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
11372 make_unused_array (hole, size);
11373 generation_free_obj_space (gen) += size;
11378 dprintf (3, ("threading hole in front of free list"));
11379 make_unused_array (hole, size);
11380 generation_free_list_space (gen) += size;
11381 generation_allocator(gen)->thread_item_front (hole, size);
11382 add_gen_free (gen->gen_num, size);
11387 make_unused_array (hole, size);
11388 generation_free_obj_space (gen) += size;
11392 generation_allocation_pointer (gen) = start;
11393 generation_allocation_context_start_region (gen) = start;
11395 generation_allocation_limit (gen) = (start + limit_size);
11398 void verify_mem_cleared (uint8_t* start, size_t size)
11400 if (!Aligned (size))
11405 PTR_PTR curr_ptr = (PTR_PTR) start;
11406 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11408 if (*(curr_ptr++) != 0)
11415 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11416 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11418 size_t start_mark_bit = mark_bit_of (start);
11419 size_t end_mark_bit = mark_bit_of (end);
11420 unsigned int startbit = mark_bit_bit (start_mark_bit);
11421 unsigned int endbit = mark_bit_bit (end_mark_bit);
11422 size_t startwrd = mark_bit_word (start_mark_bit);
11423 size_t endwrd = mark_bit_word (end_mark_bit);
11425 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11426 (size_t)start, (size_t)start_mark_bit,
11427 (size_t)end, (size_t)end_mark_bit));
11429 unsigned int firstwrd = ~(lowbits (~0, startbit));
11430 unsigned int lastwrd = ~(highbits (~0, endbit));
11432 if (startwrd == endwrd)
11434 unsigned int wrd = firstwrd & lastwrd;
11435 mark_array[startwrd] |= wrd;
11439 // set the first mark word.
11442 mark_array[startwrd] |= firstwrd;
11446 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11448 mark_array[wrdtmp] = ~(unsigned int)0;
11451 // set the last mark word.
11454 mark_array[endwrd] |= lastwrd;
11458 // makes sure that the mark array bits between start and end are 0.
11459 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11461 size_t start_mark_bit = mark_bit_of (start);
11462 size_t end_mark_bit = mark_bit_of (end);
11463 unsigned int startbit = mark_bit_bit (start_mark_bit);
11464 unsigned int endbit = mark_bit_bit (end_mark_bit);
11465 size_t startwrd = mark_bit_word (start_mark_bit);
11466 size_t endwrd = mark_bit_word (end_mark_bit);
11468 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11469 // (size_t)start, (size_t)start_mark_bit,
11470 // (size_t)end, (size_t)end_mark_bit));
11472 unsigned int firstwrd = ~(lowbits (~0, startbit));
11473 unsigned int lastwrd = ~(highbits (~0, endbit));
11475 if (startwrd == endwrd)
11477 unsigned int wrd = firstwrd & lastwrd;
11478 if (mark_array[startwrd] & wrd)
11480 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11482 mark_array [startwrd], mark_word_address (startwrd)));
11488 // set the first mark word.
11491 if (mark_array[startwrd] & firstwrd)
11493 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11494 firstwrd, startwrd,
11495 mark_array [startwrd], mark_word_address (startwrd)));
11502 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11504 if (mark_array[wrdtmp])
11506 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11508 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11513 // set the last mark word.
11516 if (mark_array[endwrd] & lastwrd)
11518 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11520 mark_array [lastwrd], mark_word_address (lastwrd)));
11525 #endif //VERIFY_HEAP && BACKGROUND_GC
11527 allocator::allocator (unsigned int num_b, int fbb, alloc_list* b)
11529 assert (num_b < MAX_BUCKET_COUNT);
11530 num_buckets = num_b;
11531 first_bucket_bits = fbb;
11535 alloc_list& allocator::alloc_list_of (unsigned int bn)
11537 assert (bn < num_buckets);
11539 return first_bucket;
11541 return buckets [bn-1];
11544 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11546 assert (bn < num_buckets);
11548 return first_bucket.alloc_list_damage_count();
11550 return buckets [bn-1].alloc_list_damage_count();
11553 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11555 //unlink the free_item
11556 alloc_list* al = &alloc_list_of (bn);
11559 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11561 assert (item == free_list_slot (prev_item));
11562 free_list_undo (prev_item) = item;
11563 alloc_list_damage_count_of (bn)++;
11565 free_list_slot (prev_item) = free_list_slot(item);
11569 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11571 if (al->alloc_list_tail() == item)
11573 al->alloc_list_tail() = prev_item;
11577 void allocator::clear()
11579 for (unsigned int i = 0; i < num_buckets; i++)
11581 alloc_list_head_of (i) = 0;
11582 alloc_list_tail_of (i) = 0;
11586 //always thread to the end.
11587 void allocator::thread_item (uint8_t* item, size_t size)
11589 unsigned int a_l_number = first_suitable_bucket(size);
11590 alloc_list* al = &alloc_list_of (a_l_number);
11591 uint8_t*& head = al->alloc_list_head();
11592 uint8_t*& tail = al->alloc_list_tail();
11594 free_list_slot (item) = 0;
11595 free_list_undo (item) = UNDO_EMPTY;
11596 assert (item != head);
11604 assert ((free_list_slot(head) != 0) || (tail == head));
11605 assert (item != tail);
11606 assert (free_list_slot(tail) == 0);
11608 free_list_slot (tail) = item;
11614 void allocator::thread_item_front (uint8_t* item, size_t size)
11616 unsigned int a_l_number = first_suitable_bucket (size);
11617 alloc_list* al = &alloc_list_of (a_l_number);
11619 free_list_slot (item) = al->alloc_list_head();
11620 free_list_undo (item) = UNDO_EMPTY;
11622 if (al->alloc_list_tail() == 0)
11624 al->alloc_list_tail() = al->alloc_list_head();
11626 al->alloc_list_head() = item;
11627 if (al->alloc_list_tail() == 0)
11629 al->alloc_list_tail() = item;
11633 void allocator::copy_to_alloc_list (alloc_list* toalist)
11635 for (unsigned int i = 0; i < num_buckets; i++)
11637 toalist [i] = alloc_list_of (i);
11638 #ifdef FL_VERIFICATION
11639 uint8_t* free_item = alloc_list_head_of (i);
11644 free_item = free_list_slot (free_item);
11647 toalist[i].item_count = count;
11648 #endif //FL_VERIFICATION
11652 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11654 BOOL repair_list = !discard_if_no_fit_p ();
11655 for (unsigned int i = 0; i < num_buckets; i++)
11657 size_t count = alloc_list_damage_count_of (i);
11658 alloc_list_of (i) = fromalist [i];
11659 assert (alloc_list_damage_count_of (i) == 0);
11663 //repair the the list
11664 //new items may have been added during the plan phase
11665 //items may have been unlinked.
11666 uint8_t* free_item = alloc_list_head_of (i);
11667 while (free_item && count)
11669 assert (((CObjectHeader*)free_item)->IsFree());
11670 if ((free_list_undo (free_item) != UNDO_EMPTY))
11673 free_list_slot (free_item) = free_list_undo (free_item);
11674 free_list_undo (free_item) = UNDO_EMPTY;
11677 free_item = free_list_slot (free_item);
11680 #ifdef FL_VERIFICATION
11681 free_item = alloc_list_head_of (i);
11682 size_t item_count = 0;
11686 free_item = free_list_slot (free_item);
11689 assert (item_count == alloc_list_of (i).item_count);
11690 #endif //FL_VERIFICATION
11693 uint8_t* tail_item = alloc_list_tail_of (i);
11694 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11699 void allocator::commit_alloc_list_changes()
11701 BOOL repair_list = !discard_if_no_fit_p ();
11704 for (unsigned int i = 0; i < num_buckets; i++)
11706 //remove the undo info from list.
11707 uint8_t* free_item = alloc_list_head_of (i);
11708 size_t count = alloc_list_damage_count_of (i);
11709 while (free_item && count)
11711 assert (((CObjectHeader*)free_item)->IsFree());
11713 if (free_list_undo (free_item) != UNDO_EMPTY)
11715 free_list_undo (free_item) = UNDO_EMPTY;
11719 free_item = free_list_slot (free_item);
11722 alloc_list_damage_count_of (i) = 0;
11727 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size, size_t size,
11728 alloc_context* acontext, uint32_t flags,
11729 heap_segment* seg, int align_const, int gen_number)
11731 bool uoh_p = (gen_number > 0);
11732 GCSpinLock* msl = uoh_p ? &more_space_lock_uoh : &more_space_lock_soh;
11733 uint64_t& total_alloc_bytes = uoh_p ? total_alloc_bytes_uoh : total_alloc_bytes_soh;
11735 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11739 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11742 #ifdef MULTIPLE_HEAPS
11743 if (gen_number == 0)
11745 if (!gen0_allocated_after_gc_p)
11747 gen0_allocated_after_gc_p = true;
11750 #endif //MULTIPLE_HEAPS
11752 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11753 (size_t)start + limit_size - aligned_min_obj_size));
11755 if ((acontext->alloc_limit != start) &&
11756 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11758 uint8_t* hole = acontext->alloc_ptr;
11761 size_t ac_size = (acontext->alloc_limit - acontext->alloc_ptr);
11762 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + ac_size + Align (min_obj_size, align_const)));
11763 // when we are finishing an allocation from a free list
11764 // we know that the free area was Align(min_obj_size) larger
11765 acontext->alloc_bytes -= ac_size;
11766 total_alloc_bytes -= ac_size;
11767 size_t free_obj_size = ac_size + aligned_min_obj_size;
11768 make_unused_array (hole, free_obj_size);
11769 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11771 acontext->alloc_ptr = start;
11775 if (gen_number == 0)
11777 size_t pad_size = Align (min_obj_size, align_const);
11778 dprintf (3, ("contigous ac: making min obj gap %Ix->%Ix(%Id)",
11779 acontext->alloc_ptr, (acontext->alloc_ptr + pad_size), pad_size));
11780 make_unused_array (acontext->alloc_ptr, pad_size);
11781 acontext->alloc_ptr += pad_size;
11784 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11785 size_t added_bytes = limit_size - ((gen_number <= max_generation) ? aligned_min_obj_size : 0);
11786 acontext->alloc_bytes += added_bytes;
11787 total_alloc_bytes += added_bytes;
11789 uint8_t* saved_used = 0;
11793 saved_used = heap_segment_used (seg);
11796 if (seg == ephemeral_heap_segment)
11798 //Sometimes the allocated size is advanced without clearing the
11799 //memory. Let's catch up here
11800 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11802 heap_segment_used (seg) = alloc_allocated - plug_skew;
11805 #ifdef BACKGROUND_GC
11808 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11809 #ifdef FEATURE_LOH_COMPACTION
11810 if (gen_number == loh_generation)
11812 old_allocated -= Align (loh_padding_obj_size, align_const);
11814 #endif //FEATURE_LOH_COMPACTION
11816 assert (heap_segment_used (seg) >= old_allocated);
11818 #endif //BACKGROUND_GC
11820 // we are going to clear a right-edge exclusive span [clear_start, clear_limit)
11821 // but will adjust for cases when object is ok to stay dirty or the space has not seen any use yet
11822 // NB: the size and limit_size include syncblock, which is to the -1 of the object start
11823 // that effectively shifts the allocation by `plug_skew`
11824 uint8_t* clear_start = start - plug_skew;
11825 uint8_t* clear_limit = start + limit_size - plug_skew;
11827 if (flags & GC_ALLOC_ZEROING_OPTIONAL)
11829 uint8_t* obj_start = acontext->alloc_ptr;
11830 assert(start >= obj_start);
11831 uint8_t* obj_end = obj_start + size - plug_skew;
11832 assert(obj_end >= clear_start);
11834 // if clearing at the object start, clear the syncblock.
11835 if(obj_start == start)
11837 *(PTR_PTR)clear_start = 0;
11839 // skip the rest of the object
11840 dprintf(3, ("zeroing optional: skipping object at %Ix->%Ix(%Id)", clear_start, obj_end, obj_end - clear_start));
11841 clear_start = obj_end;
11844 // check if space to clear is all dirty from prior use or only partially
11845 if ((seg == 0) || (clear_limit <= heap_segment_used (seg)))
11847 add_saved_spinlock_info (uoh_p, me_release, mt_clr_mem);
11848 leave_spin_lock (msl);
11850 if (clear_start < clear_limit)
11852 dprintf(3, ("clearing memory at %Ix for %d bytes", clear_start, clear_limit - clear_start));
11853 memclr(clear_start, clear_limit - clear_start);
11858 // we only need to clear [clear_start, used) and only if clear_start < used
11859 uint8_t* used = heap_segment_used (seg);
11860 heap_segment_used (seg) = clear_limit;
11862 add_saved_spinlock_info (uoh_p, me_release, mt_clr_mem);
11863 leave_spin_lock (msl);
11865 if (clear_start < used)
11867 if (used != saved_used)
11872 dprintf (2, ("clearing memory before used at %Ix for %Id bytes", clear_start, used - clear_start));
11873 memclr (clear_start, used - clear_start);
11877 //this portion can be done after we release the lock
11878 if (seg == ephemeral_heap_segment ||
11879 ((seg == nullptr) && (gen_number == 0) && (limit_size >= CLR_SIZE / 2)))
11881 if (gen0_must_clear_bricks > 0)
11883 //set the brick table to speed up find_object
11884 size_t b = brick_of (acontext->alloc_ptr);
11885 set_brick (b, acontext->alloc_ptr - brick_address (b));
11887 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11888 b, brick_of (align_on_brick (start + limit_size))));
11889 volatile short* x = &brick_table [b];
11890 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11892 for (;x < end_x;x++)
11897 gen0_bricks_cleared = FALSE;
11901 // verifying the memory is completely cleared.
11902 //if (!(flags & GC_ALLOC_ZEROING_OPTIONAL))
11904 // verify_mem_cleared(start - plug_skew, limit_size);
11908 size_t gc_heap::new_allocation_limit (size_t size, size_t physical_limit, int gen_number)
11910 dynamic_data* dd = dynamic_data_of (gen_number);
11911 ptrdiff_t new_alloc = dd_new_allocation (dd);
11912 assert (new_alloc == (ptrdiff_t)Align (new_alloc, get_alignment_constant (gen_number < uoh_start_generation)));
11914 ptrdiff_t logical_limit = max (new_alloc, (ptrdiff_t)size);
11915 size_t limit = min (logical_limit, (ptrdiff_t)physical_limit);
11916 assert (limit == Align (limit, get_alignment_constant (gen_number <= max_generation)));
11917 dd_new_allocation (dd) = (new_alloc - limit);
11922 size_t gc_heap::limit_from_size (size_t size, uint32_t flags, size_t physical_limit, int gen_number,
11925 size_t padded_size = size + Align (min_obj_size, align_const);
11926 // for LOH this is not true...we could select a physical_limit that's exactly the same
11928 assert ((gen_number != 0) || (physical_limit >= padded_size));
11930 // For SOH if the size asked for is very small, we want to allocate more than just what's asked for if possible.
11931 // Unless we were told not to clean, then we will not force it.
11932 size_t min_size_to_allocate = ((gen_number == 0 && !(flags & GC_ALLOC_ZEROING_OPTIONAL)) ? allocation_quantum : 0);
11934 size_t desired_size_to_allocate = max (padded_size, min_size_to_allocate);
11935 size_t new_physical_limit = min (physical_limit, desired_size_to_allocate);
11937 size_t new_limit = new_allocation_limit (padded_size,
11938 new_physical_limit,
11940 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11941 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11945 void gc_heap::add_to_oom_history_per_heap()
11947 oom_history* current_hist = &oomhist_per_heap[oomhist_index_per_heap];
11948 memcpy (current_hist, &oom_info, sizeof (oom_info));
11949 oomhist_index_per_heap++;
11950 if (oomhist_index_per_heap == max_oom_history_count)
11952 oomhist_index_per_heap = 0;
11956 void gc_heap::handle_oom (oom_reason reason, size_t alloc_size,
11957 uint8_t* allocated, uint8_t* reserved)
11959 if (reason == oom_budget)
11961 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11964 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11966 // This means during the last GC we needed to reserve and/or commit more memory
11967 // but we couldn't. We proceeded with the GC and ended up not having enough
11968 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11969 // probably made a mistake and didn't expand the heap when we should have.
11970 reason = oom_low_mem;
11973 oom_info.reason = reason;
11974 oom_info.allocated = allocated;
11975 oom_info.reserved = reserved;
11976 oom_info.alloc_size = alloc_size;
11977 oom_info.gc_index = settings.gc_index;
11978 oom_info.fgm = fgm_result.fgm;
11979 oom_info.size = fgm_result.size;
11980 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11981 oom_info.loh_p = fgm_result.loh_p;
11983 add_to_oom_history_per_heap();
11984 fgm_result.fgm = fgm_no_failure;
11986 // Break early - before the more_space_lock is release so no other threads
11987 // could have allocated on the same heap when OOM happened.
11988 if (GCConfig::GetBreakOnOOM())
11990 GCToOSInterface::DebugBreak();
11994 #ifdef BACKGROUND_GC
11995 BOOL gc_heap::background_allowed_p()
11997 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11999 #endif //BACKGROUND_GC
12001 void gc_heap::check_for_full_gc (int gen_num, size_t size)
12003 BOOL should_notify = FALSE;
12004 // if we detect full gc because of the allocation budget specified this is TRUE;
12005 // it's FALSE if it's due to other factors.
12006 BOOL alloc_factor = TRUE;
12007 int n_initial = gen_num;
12008 BOOL local_blocking_collection = FALSE;
12009 BOOL local_elevation_requested = FALSE;
12010 int new_alloc_remain_percent = 0;
12012 if (full_gc_approach_event_set)
12017 if (gen_num < max_generation)
12019 gen_num = max_generation;
12022 dynamic_data* dd_full = dynamic_data_of (gen_num);
12023 ptrdiff_t new_alloc_remain = 0;
12024 uint32_t pct = (gen_num >= uoh_start_generation) ? fgn_loh_percent : fgn_maxgen_percent;
12026 for (int gen_index = 0; gen_index < total_generation_count; gen_index++)
12028 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
12029 heap_number, gen_index,
12030 dd_new_allocation (dynamic_data_of (gen_index)),
12031 dd_desired_allocation (dynamic_data_of (gen_index))));
12034 // For small object allocations we only check every fgn_check_quantum bytes.
12035 if (n_initial == 0)
12037 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
12038 dynamic_data* dd_0 = dynamic_data_of (n_initial);
12039 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
12040 (dd_new_allocation (dd_0) >= 0))
12046 fgn_last_alloc = dd_new_allocation (dd_0);
12047 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
12050 // We don't consider the size that came from soh 'cause it doesn't contribute to the
12056 for (int i = 1; i <= max_generation; i++)
12058 if (get_new_allocation (i) <= 0)
12066 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
12067 if (gen_num == max_generation)
12069 // If it's small object heap we should first see if we will even be looking at gen2 budget
12070 // in the next GC or not. If not we should go directly to checking other factors.
12071 if (n < (max_generation - 1))
12073 goto check_other_factors;
12077 new_alloc_remain = dd_new_allocation (dd_full) - size;
12079 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
12081 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
12082 gen_num, pct, new_alloc_remain_percent));
12084 if (new_alloc_remain_percent <= (int)pct)
12086 #ifdef BACKGROUND_GC
12087 // If background GC is enabled, we still want to check whether this will
12088 // be a blocking GC or not because we only want to notify when it's a
12089 // blocking full GC.
12090 if (background_allowed_p())
12092 goto check_other_factors;
12094 #endif //BACKGROUND_GC
12096 should_notify = TRUE;
12100 check_other_factors:
12102 dprintf (2, ("FGC: checking other factors"));
12103 n = generation_to_condemn (n,
12104 &local_blocking_collection,
12105 &local_elevation_requested,
12108 if (local_elevation_requested && (n == max_generation))
12110 if (settings.should_lock_elevation)
12112 int local_elevation_locked_count = settings.elevation_locked_count + 1;
12113 if (local_elevation_locked_count != 6)
12115 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
12116 local_elevation_locked_count));
12117 n = max_generation - 1;
12122 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
12124 #ifdef BACKGROUND_GC
12125 // When background GC is enabled it decreases the accuracy of our predictability -
12126 // by the time the GC happens, we may not be under BGC anymore. If we try to
12127 // predict often enough it should be ok.
12128 if ((n == max_generation) &&
12129 (gc_heap::background_running_p()))
12131 n = max_generation - 1;
12132 dprintf (2, ("FGN: bgc - 1 instead of 2"));
12135 if ((n == max_generation) && !local_blocking_collection)
12137 if (!background_allowed_p())
12139 local_blocking_collection = TRUE;
12142 #endif //BACKGROUND_GC
12144 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
12146 (local_blocking_collection ? "blocking" : "background")));
12148 if ((n == max_generation) && local_blocking_collection)
12150 alloc_factor = FALSE;
12151 should_notify = TRUE;
12159 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
12161 (alloc_factor ? "alloc" : "other"),
12162 dd_collection_count (dynamic_data_of (0)),
12163 new_alloc_remain_percent,
12166 send_full_gc_notification (n_initial, alloc_factor);
12170 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
12172 if (!full_gc_approach_event_set)
12174 assert (full_gc_approach_event.IsValid());
12175 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
12177 full_gc_end_event.Reset();
12178 full_gc_approach_event.Set();
12179 full_gc_approach_event_set = true;
12183 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
12185 #ifdef MULTIPLE_HEAPS
12186 gc_heap* hp = gc_heap::g_heaps[0];
12188 gc_heap* hp = pGenGCHeap;
12189 #endif //MULTIPLE_HEAPS
12191 if (hp->fgn_maxgen_percent == 0)
12193 return wait_full_gc_na;
12196 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
12198 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
12200 if (hp->fgn_maxgen_percent == 0)
12202 return wait_full_gc_cancelled;
12205 if (wait_result == WAIT_OBJECT_0)
12207 #ifdef BACKGROUND_GC
12208 if (fgn_last_gc_was_concurrent)
12210 fgn_last_gc_was_concurrent = FALSE;
12211 return wait_full_gc_na;
12214 #endif //BACKGROUND_GC
12216 return wait_full_gc_success;
12221 return wait_full_gc_timeout;
12226 return wait_full_gc_failed;
12230 size_t gc_heap::get_full_compact_gc_count()
12232 return full_gc_counts[gc_type_compacting];
12235 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
12238 BOOL gc_heap::short_on_end_of_seg (heap_segment* seg, int align_const)
12240 uint8_t* allocated = heap_segment_allocated(seg);
12242 BOOL sufficient_p = sufficient_space_end_seg (allocated,
12243 heap_segment_reserved (seg),
12244 end_space_after_gc(),
12245 tuning_deciding_short_on_seg);
12248 if (sufficient_gen0_space_p)
12250 dprintf (GTC_LOG, ("gen0 has enough free space"));
12253 sufficient_p = sufficient_gen0_space_p;
12256 return !sufficient_p;
12260 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
12264 BOOL gc_heap::a_fit_free_list_p (int gen_number,
12266 alloc_context* acontext,
12270 BOOL can_fit = FALSE;
12271 generation* gen = generation_of (gen_number);
12272 allocator* gen_allocator = generation_allocator (gen);
12274 for (unsigned int a_l_idx = gen_allocator->first_suitable_bucket(size); a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
12276 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
12277 uint8_t* prev_free_item = 0;
12279 while (free_list != 0)
12281 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12282 size_t free_list_size = unused_array_size (free_list);
12283 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
12285 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
12286 (size_t)free_list, free_list_size));
12288 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12289 // We ask for more Align (min_obj_size)
12290 // to make sure that we can insert a free object
12291 // in adjust_limit will set the limit lower
12292 size_t limit = limit_from_size (size, flags, free_list_size, gen_number, align_const);
12294 uint8_t* remain = (free_list + limit);
12295 size_t remain_size = (free_list_size - limit);
12296 if (remain_size >= Align(min_free_list, align_const))
12298 make_unused_array (remain, remain_size);
12299 gen_allocator->thread_item_front (remain, remain_size);
12300 assert (remain_size >= Align (min_obj_size, align_const));
12304 //absorb the entire free list
12305 limit += remain_size;
12307 generation_free_list_space (gen) -= limit;
12309 adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
12314 else if (gen_allocator->discard_if_no_fit_p())
12316 assert (prev_free_item == 0);
12317 dprintf (3, ("couldn't use this free area, discarding"));
12318 generation_free_obj_space (gen) += free_list_size;
12320 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12321 generation_free_list_space (gen) -= free_list_size;
12325 prev_free_item = free_list;
12327 free_list = free_list_slot (free_list);
12335 #ifdef BACKGROUND_GC
12336 void gc_heap::bgc_uoh_alloc_clr (uint8_t* alloc_start,
12338 alloc_context* acontext,
12345 make_unused_array (alloc_start, size);
12347 size_t size_of_array_base = sizeof(ArrayBase);
12349 bgc_alloc_lock->uoh_alloc_done_with_index (lock_index);
12351 // clear memory while not holding the lock.
12352 size_t size_to_skip = size_of_array_base;
12353 size_t size_to_clear = size - size_to_skip - plug_skew;
12354 size_t saved_size_to_clear = size_to_clear;
12357 uint8_t* end = alloc_start + size - plug_skew;
12358 uint8_t* used = heap_segment_used (seg);
12361 if ((alloc_start + size_to_skip) < used)
12363 size_to_clear = used - (alloc_start + size_to_skip);
12369 dprintf (2, ("bgc uoh: setting used to %Ix", end));
12370 heap_segment_used (seg) = end;
12373 dprintf (2, ("bgc uoh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
12374 used, alloc_start, end, size_to_clear));
12378 dprintf (2, ("bgc uoh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
12382 // since we filled in 0xcc for free object when we verify heap,
12383 // we need to make sure we clear those bytes.
12384 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
12386 if (size_to_clear < saved_size_to_clear)
12388 size_to_clear = saved_size_to_clear;
12391 #endif //VERIFY_HEAP
12393 total_alloc_bytes_uoh += size - Align (min_obj_size, align_const);
12395 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear uoh obj", heap_number));
12396 add_saved_spinlock_info (true, me_release, mt_clr_large_mem);
12397 leave_spin_lock (&more_space_lock_uoh);
12399 ((void**) alloc_start)[-1] = 0; //clear the sync block
12400 if (!(flags & GC_ALLOC_ZEROING_OPTIONAL))
12402 memclr(alloc_start + size_to_skip, size_to_clear);
12405 bgc_alloc_lock->uoh_alloc_set (alloc_start);
12407 acontext->alloc_ptr = alloc_start;
12408 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
12410 // need to clear the rest of the object before we hand it out.
12411 clear_unused_array(alloc_start, size);
12413 #endif //BACKGROUND_GC
12415 BOOL gc_heap::a_fit_free_list_uoh_p (size_t size,
12416 alloc_context* acontext,
12421 BOOL can_fit = FALSE;
12422 generation* gen = generation_of (gen_number);
12423 allocator* allocator = generation_allocator (gen);
12425 #ifdef FEATURE_LOH_COMPACTION
12426 size_t loh_pad = gen_number == loh_generation ? Align (loh_padding_obj_size, align_const) : 0;
12427 #endif //FEATURE_LOH_COMPACTION
12429 #ifdef BACKGROUND_GC
12431 #endif //BACKGROUND_GC
12433 for (unsigned int a_l_idx = allocator->first_suitable_bucket(size); a_l_idx < allocator->number_of_buckets(); a_l_idx++)
12435 uint8_t* free_list = allocator->alloc_list_head_of (a_l_idx);
12436 uint8_t* prev_free_item = 0;
12437 while (free_list != 0)
12439 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12441 size_t free_list_size = unused_array_size(free_list);
12443 ptrdiff_t diff = free_list_size - size;
12445 #ifdef FEATURE_LOH_COMPACTION
12447 #endif //FEATURE_LOH_COMPACTION
12449 // must fit exactly or leave formattable space
12450 if ((diff == 0) || (diff >= (ptrdiff_t)Align (min_obj_size, align_const)))
12452 #ifdef BACKGROUND_GC
12453 cookie = bgc_alloc_lock->uoh_alloc_set (free_list);
12454 bgc_track_uoh_alloc();
12455 #endif //BACKGROUND_GC
12457 //unlink the free_item
12458 allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12460 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12461 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), flags, free_list_size,
12462 gen_number, align_const);
12464 #ifdef FEATURE_LOH_COMPACTION
12467 make_unused_array (free_list, loh_pad);
12469 free_list += loh_pad;
12470 free_list_size -= loh_pad;
12472 #endif //FEATURE_LOH_COMPACTION
12474 uint8_t* remain = (free_list + limit);
12475 size_t remain_size = (free_list_size - limit);
12476 if (remain_size != 0)
12478 assert (remain_size >= Align (min_obj_size, align_const));
12479 make_unused_array (remain, remain_size);
12481 if (remain_size >= Align(min_free_list, align_const))
12483 loh_thread_gap_front (remain, remain_size, gen);
12484 assert (remain_size >= Align (min_obj_size, align_const));
12488 generation_free_obj_space (gen) += remain_size;
12490 generation_free_list_space (gen) -= free_list_size;
12491 dprintf (3, ("found fit on loh at %Ix", free_list));
12492 #ifdef BACKGROUND_GC
12495 bgc_uoh_alloc_clr (free_list, limit, acontext, flags, align_const, cookie, FALSE, 0);
12498 #endif //BACKGROUND_GC
12500 adjust_limit_clr (free_list, limit, size, acontext, flags, 0, align_const, gen_number);
12503 //fix the limit to compensate for adjust_limit_clr making it too short
12504 acontext->alloc_limit += Align (min_obj_size, align_const);
12508 prev_free_item = free_list;
12509 free_list = free_list_slot (free_list);
12517 #pragma warning(default:4706)
12520 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12523 alloc_context* acontext,
12526 BOOL* commit_failed_p)
12528 *commit_failed_p = FALSE;
12530 bool hard_limit_short_seg_end_p = false;
12531 #ifdef BACKGROUND_GC
12533 #endif //BACKGROUND_GC
12535 uint8_t*& allocated = ((gen_number == 0) ?
12537 heap_segment_allocated(seg));
12539 size_t pad = Align (min_obj_size, align_const);
12541 #ifdef FEATURE_LOH_COMPACTION
12542 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12543 if (gen_number == loh_generation)
12547 #endif //FEATURE_LOH_COMPACTION
12549 uint8_t* end = heap_segment_committed (seg) - pad;
12551 if (a_size_fit_p (size, allocated, end, align_const))
12553 limit = limit_from_size (size,
12556 gen_number, align_const);
12560 end = heap_segment_reserved (seg) - pad;
12562 if (a_size_fit_p (size, allocated, end, align_const))
12564 limit = limit_from_size (size,
12567 gen_number, align_const);
12569 if (grow_heap_segment (seg, (allocated + limit), &hard_limit_short_seg_end_p))
12575 if (!hard_limit_short_seg_end_p)
12577 dprintf (2, ("can't grow segment, doing a full gc"));
12578 *commit_failed_p = TRUE;
12582 assert (heap_hard_limit);
12591 #ifdef BACKGROUND_GC
12592 if (gen_number != 0)
12594 cookie = bgc_alloc_lock->uoh_alloc_set (allocated);
12595 bgc_track_uoh_alloc();
12597 #endif //BACKGROUND_GC
12599 #ifdef FEATURE_LOH_COMPACTION
12600 if (gen_number == loh_generation)
12602 make_unused_array (allocated, loh_pad);
12603 allocated += loh_pad;
12606 #endif //FEATURE_LOH_COMPACTION
12608 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12609 // we are responsible for cleaning the syncblock and we will do it later
12610 // as a part of cleanup routine and when not holding the heap lock.
12611 // However, once we move "allocated" forward and if another thread initiate verification of
12612 // the previous object, it may consider the syncblock in the "next" eligible for validation.
12613 // (see also: object.cpp/Object::ValidateInner)
12614 // Make sure it will see cleaned up state to prevent triggering occasional verification failures.
12615 // And make sure the write happens before updating "allocated"
12616 VolatileStore(((void**)allocated - 1), (void*)0); //clear the sync block
12617 #endif //VERIFY_HEAP && _DEBUG
12619 uint8_t* old_alloc;
12620 old_alloc = allocated;
12621 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12623 #ifdef BACKGROUND_GC
12626 allocated += limit;
12627 bgc_uoh_alloc_clr (old_alloc, limit, acontext, flags, align_const, cookie, TRUE, seg);
12630 #endif //BACKGROUND_GC
12632 // In a contiguous AC case with GC_ALLOC_ZEROING_OPTIONAL, deduct unspent space from the limit to clear only what is necessary.
12633 if ((flags & GC_ALLOC_ZEROING_OPTIONAL) &&
12634 ((allocated == acontext->alloc_limit) || (allocated == (acontext->alloc_limit + Align (min_obj_size, align_const)))))
12636 assert(gen_number == 0);
12637 assert(allocated > acontext->alloc_ptr);
12639 size_t extra = allocated - acontext->alloc_ptr;
12642 // Since we are not consuming all the memory we already deducted from the budget,
12643 // we should put the extra back.
12644 dynamic_data* dd = dynamic_data_of (0);
12645 dd_new_allocation (dd) += extra;
12647 // add space for an AC continuity divider
12648 limit += Align(min_obj_size, align_const);
12651 allocated += limit;
12652 adjust_limit_clr (old_alloc, limit, size, acontext, flags, seg, align_const, gen_number);
12662 BOOL gc_heap::uoh_a_fit_segment_end_p (int gen_number,
12664 alloc_context* acontext,
12667 BOOL* commit_failed_p,
12670 *commit_failed_p = FALSE;
12671 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12672 BOOL can_allocate_p = FALSE;
12676 #ifdef BACKGROUND_GC
12677 if (seg->flags & heap_segment_flags_uoh_delete)
12679 dprintf (3, ("h%d skipping seg %Ix to be deleted", heap_number, (size_t)seg));
12682 #endif //BACKGROUND_GC
12684 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12685 acontext, flags, align_const, commit_failed_p))
12687 acontext->alloc_limit += Align (min_obj_size, align_const);
12688 can_allocate_p = TRUE;
12692 if (*commit_failed_p)
12694 *oom_r = oom_cant_commit;
12699 seg = heap_segment_next_rw (seg);
12702 return can_allocate_p;
12705 #ifdef BACKGROUND_GC
12707 void gc_heap::wait_for_background (alloc_wait_reason awr, bool loh_p)
12709 GCSpinLock* msl = loh_p ? &more_space_lock_uoh : &more_space_lock_soh;
12711 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12712 add_saved_spinlock_info (loh_p, me_release, mt_wait_bgc);
12713 leave_spin_lock (msl);
12714 background_gc_wait (awr);
12715 enter_spin_lock (msl);
12716 add_saved_spinlock_info (loh_p, me_acquire, mt_wait_bgc);
12719 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr, bool loh_p)
12721 if (gc_heap::background_running_p())
12723 uint32_t memory_load;
12724 get_memory_info (&memory_load);
12725 if (memory_load >= m_high_memory_load_th)
12727 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12728 wait_for_background (awr, loh_p);
12733 #endif //BACKGROUND_GC
12735 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12736 // return TRUE if that's the case.
12737 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12739 #ifdef BACKGROUND_GC
12740 wait_for_bgc_high_memory (awr_loh_oos_bgc, false);
12741 #endif //BACKGROUND_GC
12743 BOOL did_full_compact_gc = FALSE;
12745 dprintf (1, ("h%d triggering a gen1 GC", heap_number));
12746 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12747 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12749 #ifdef MULTIPLE_HEAPS
12750 enter_spin_lock (&more_space_lock_soh);
12751 add_saved_spinlock_info (false, me_acquire, mt_t_eph_gc);
12752 #endif //MULTIPLE_HEAPS
12754 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12756 if (current_full_compact_gc_count > last_full_compact_gc_count)
12758 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12759 did_full_compact_gc = TRUE;
12762 return did_full_compact_gc;
12765 BOOL gc_heap::soh_try_fit (int gen_number,
12767 alloc_context* acontext,
12770 BOOL* commit_failed_p,
12771 BOOL* short_seg_end_p)
12773 BOOL can_allocate = TRUE;
12774 if (short_seg_end_p)
12776 *short_seg_end_p = FALSE;
12779 can_allocate = a_fit_free_list_p (gen_number, size, acontext, flags, align_const);
12782 if (short_seg_end_p)
12784 *short_seg_end_p = short_on_end_of_seg (ephemeral_heap_segment, align_const);
12786 // If the caller doesn't care, we always try to fit at the end of seg;
12787 // otherwise we would only try if we are actually not short at end of seg.
12788 if (!short_seg_end_p || !(*short_seg_end_p))
12790 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12791 acontext, flags, align_const, commit_failed_p);
12795 return can_allocate;
12798 allocation_state gc_heap::allocate_soh (int gen_number,
12800 alloc_context* acontext,
12804 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12805 if (gc_heap::background_running_p())
12807 background_soh_alloc_count++;
12808 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12810 add_saved_spinlock_info (false, me_release, mt_alloc_small);
12811 leave_spin_lock (&more_space_lock_soh);
12812 bool cooperative_mode = enable_preemptive();
12813 GCToOSInterface::Sleep (bgc_alloc_spin);
12814 disable_preemptive (cooperative_mode);
12815 enter_spin_lock (&more_space_lock_soh);
12816 add_saved_spinlock_info (false, me_acquire, mt_alloc_small);
12820 //GCToOSInterface::YieldThread (0);
12823 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12825 gc_reason gr = reason_oos_soh;
12826 oom_reason oom_r = oom_no_failure;
12828 // No variable values should be "carried over" from one state to the other.
12829 // That's why there are local variable for each state
12831 allocation_state soh_alloc_state = a_state_start;
12833 // If we can get a new seg it means allocation will succeed.
12836 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12838 switch (soh_alloc_state)
12840 case a_state_can_allocate:
12841 case a_state_cant_allocate:
12845 case a_state_start:
12847 soh_alloc_state = a_state_try_fit;
12850 case a_state_try_fit:
12852 BOOL commit_failed_p = FALSE;
12853 BOOL can_use_existing_p = FALSE;
12855 can_use_existing_p = soh_try_fit (gen_number, size, acontext, flags,
12856 align_const, &commit_failed_p,
12858 soh_alloc_state = (can_use_existing_p ?
12859 a_state_can_allocate :
12861 a_state_trigger_full_compact_gc :
12862 a_state_trigger_ephemeral_gc));
12865 case a_state_try_fit_after_bgc:
12867 BOOL commit_failed_p = FALSE;
12868 BOOL can_use_existing_p = FALSE;
12869 BOOL short_seg_end_p = FALSE;
12871 can_use_existing_p = soh_try_fit (gen_number, size, acontext, flags,
12872 align_const, &commit_failed_p,
12874 soh_alloc_state = (can_use_existing_p ?
12875 a_state_can_allocate :
12877 a_state_trigger_2nd_ephemeral_gc :
12878 a_state_trigger_full_compact_gc));
12881 case a_state_try_fit_after_cg:
12883 BOOL commit_failed_p = FALSE;
12884 BOOL can_use_existing_p = FALSE;
12885 BOOL short_seg_end_p = FALSE;
12887 can_use_existing_p = soh_try_fit (gen_number, size, acontext, flags,
12888 align_const, &commit_failed_p,
12891 if (can_use_existing_p)
12893 soh_alloc_state = a_state_can_allocate;
12895 #ifdef MULTIPLE_HEAPS
12896 else if (gen0_allocated_after_gc_p)
12898 // some other threads already grabbed the more space lock and allocated
12899 // so we should attempt an ephemeral GC again.
12900 soh_alloc_state = a_state_trigger_ephemeral_gc;
12902 #endif //MULTIPLE_HEAPS
12903 else if (short_seg_end_p)
12905 soh_alloc_state = a_state_cant_allocate;
12906 oom_r = oom_budget;
12910 assert (commit_failed_p || heap_hard_limit);
12911 soh_alloc_state = a_state_cant_allocate;
12912 oom_r = oom_cant_commit;
12916 case a_state_check_and_wait_for_bgc:
12918 BOOL bgc_in_progress_p = FALSE;
12919 BOOL did_full_compacting_gc = FALSE;
12921 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc, false);
12922 soh_alloc_state = (did_full_compacting_gc ?
12923 a_state_try_fit_after_cg :
12924 a_state_try_fit_after_bgc);
12927 case a_state_trigger_ephemeral_gc:
12929 BOOL commit_failed_p = FALSE;
12930 BOOL can_use_existing_p = FALSE;
12931 BOOL short_seg_end_p = FALSE;
12932 BOOL bgc_in_progress_p = FALSE;
12933 BOOL did_full_compacting_gc = FALSE;
12935 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12936 if (did_full_compacting_gc)
12938 soh_alloc_state = a_state_try_fit_after_cg;
12942 can_use_existing_p = soh_try_fit (gen_number, size, acontext, flags,
12943 align_const, &commit_failed_p,
12945 #ifdef BACKGROUND_GC
12946 bgc_in_progress_p = gc_heap::background_running_p();
12947 #endif //BACKGROUND_GC
12949 if (can_use_existing_p)
12951 soh_alloc_state = a_state_can_allocate;
12955 if (short_seg_end_p)
12957 if (should_expand_in_full_gc)
12959 dprintf (2, ("gen1 GC wanted to expand!"));
12960 soh_alloc_state = a_state_trigger_full_compact_gc;
12964 soh_alloc_state = (bgc_in_progress_p ?
12965 a_state_check_and_wait_for_bgc :
12966 a_state_trigger_full_compact_gc);
12969 else if (commit_failed_p)
12971 soh_alloc_state = a_state_trigger_full_compact_gc;
12975 #ifdef MULTIPLE_HEAPS
12976 // some other threads already grabbed the more space lock and allocated
12977 // so we should attempt an ephemeral GC again.
12978 assert (gen0_allocated_after_gc_p);
12979 soh_alloc_state = a_state_trigger_ephemeral_gc;
12980 #else //MULTIPLE_HEAPS
12981 assert (!"shouldn't get here");
12982 #endif //MULTIPLE_HEAPS
12988 case a_state_trigger_2nd_ephemeral_gc:
12990 BOOL commit_failed_p = FALSE;
12991 BOOL can_use_existing_p = FALSE;
12992 BOOL short_seg_end_p = FALSE;
12993 BOOL did_full_compacting_gc = FALSE;
12996 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12998 if (did_full_compacting_gc)
13000 soh_alloc_state = a_state_try_fit_after_cg;
13004 can_use_existing_p = soh_try_fit (gen_number, size, acontext, flags,
13005 align_const, &commit_failed_p,
13007 if (short_seg_end_p || commit_failed_p)
13009 soh_alloc_state = a_state_trigger_full_compact_gc;
13013 assert (can_use_existing_p);
13014 soh_alloc_state = a_state_can_allocate;
13019 case a_state_trigger_full_compact_gc:
13021 if (fgn_maxgen_percent)
13023 dprintf (2, ("FGN: SOH doing last GC before we throw OOM"));
13024 send_full_gc_notification (max_generation, FALSE);
13027 BOOL got_full_compacting_gc = FALSE;
13029 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, false);
13030 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13035 assert (!"Invalid state!");
13042 if (soh_alloc_state == a_state_cant_allocate)
13044 assert (oom_r != oom_no_failure);
13047 heap_segment_allocated (ephemeral_heap_segment),
13048 heap_segment_reserved (ephemeral_heap_segment));
13050 add_saved_spinlock_info (false, me_release, mt_alloc_small_cant);
13051 leave_spin_lock (&more_space_lock_soh);
13054 assert ((soh_alloc_state == a_state_can_allocate) ||
13055 (soh_alloc_state == a_state_cant_allocate) ||
13056 (soh_alloc_state == a_state_retry_allocate));
13058 return soh_alloc_state;
13061 #ifdef BACKGROUND_GC
13063 void gc_heap::bgc_track_uoh_alloc()
13065 if (current_c_gc_state == c_gc_state_planning)
13067 Interlocked::Increment (&uoh_alloc_thread_count);
13068 dprintf (3, ("h%d: inc lc: %d", heap_number, (int32_t)uoh_alloc_thread_count));
13073 void gc_heap::bgc_untrack_uoh_alloc()
13075 if (current_c_gc_state == c_gc_state_planning)
13077 Interlocked::Decrement (&uoh_alloc_thread_count);
13078 dprintf (3, ("h%d: dec lc: %d", heap_number, (int32_t)uoh_alloc_thread_count));
13082 int bgc_allocate_spin(size_t min_gc_size, size_t bgc_begin_size, size_t bgc_size_increased, size_t end_size)
13084 if ((bgc_begin_size + bgc_size_increased) < (min_gc_size * 10))
13086 // just do it, no spinning
13090 if (((bgc_begin_size / end_size) >= 2) || (bgc_size_increased >= bgc_begin_size))
13092 if ((bgc_begin_size / end_size) >= 2)
13094 dprintf (3, ("alloc-ed too much before bgc started"));
13098 dprintf (3, ("alloc-ed too much after bgc started"));
13101 // -1 means wait for bgc
13106 return (int)(((float)bgc_size_increased / (float)bgc_begin_size) * 10);
13110 int gc_heap::bgc_loh_allocate_spin()
13112 size_t min_gc_size = dd_min_size (dynamic_data_of (loh_generation));
13113 size_t bgc_begin_size = bgc_begin_loh_size;
13114 size_t bgc_size_increased = bgc_loh_size_increased;
13115 size_t end_size = end_loh_size;
13117 return bgc_allocate_spin(min_gc_size, bgc_begin_size, bgc_size_increased, end_size);
13120 int gc_heap::bgc_poh_allocate_spin()
13122 size_t min_gc_size = dd_min_size (dynamic_data_of (poh_generation));
13123 size_t bgc_begin_size = bgc_begin_poh_size;
13124 size_t bgc_size_increased = bgc_poh_size_increased;
13125 size_t end_size = end_poh_size;
13127 return bgc_allocate_spin(min_gc_size, bgc_begin_size, bgc_size_increased, end_size);
13129 #endif //BACKGROUND_GC
13131 size_t gc_heap::get_uoh_seg_size (size_t size)
13133 size_t default_seg_size = min_uoh_segment_size;
13134 size_t align_size = default_seg_size;
13135 int align_const = get_alignment_constant (FALSE);
13136 size_t large_seg_size = align_on_page (
13137 max (default_seg_size,
13138 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
13139 align_size) / align_size * align_size)));
13140 return large_seg_size;
13143 BOOL gc_heap::uoh_get_new_seg (int gen_number,
13145 BOOL* did_full_compact_gc,
13148 *did_full_compact_gc = FALSE;
13150 size_t seg_size = get_uoh_seg_size (size);
13152 heap_segment* new_seg = get_uoh_segment (gen_number, seg_size, did_full_compact_gc);
13154 if (new_seg && gen_number == loh_generation)
13156 loh_alloc_since_cg += seg_size;
13163 return (new_seg != 0);
13166 // PERF TODO: this is too aggressive; and in hard limit we should
13167 // count the actual allocated bytes instead of only updating it during
13168 // getting a new seg.
13169 BOOL gc_heap::retry_full_compact_gc (size_t size)
13171 size_t seg_size = get_uoh_seg_size (size);
13173 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
13178 #ifdef MULTIPLE_HEAPS
13179 uint64_t total_alloc_size = 0;
13180 for (int i = 0; i < n_heaps; i++)
13182 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
13185 if (total_alloc_size >= (2 * (uint64_t)seg_size))
13189 #endif //MULTIPLE_HEAPS
13194 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
13195 BOOL* did_full_compact_gc,
13198 BOOL bgc_in_progress = FALSE;
13199 *did_full_compact_gc = FALSE;
13200 #ifdef BACKGROUND_GC
13201 if (gc_heap::background_running_p())
13203 bgc_in_progress = TRUE;
13204 size_t last_full_compact_gc_count = get_full_compact_gc_count();
13205 wait_for_background (awr, loh_p);
13206 size_t current_full_compact_gc_count = get_full_compact_gc_count();
13207 if (current_full_compact_gc_count > last_full_compact_gc_count)
13209 *did_full_compact_gc = TRUE;
13212 #endif //BACKGROUND_GC
13214 return bgc_in_progress;
13217 BOOL gc_heap::uoh_try_fit (int gen_number,
13219 alloc_context* acontext,
13222 BOOL* commit_failed_p,
13225 BOOL can_allocate = TRUE;
13227 if (!a_fit_free_list_uoh_p (size, acontext, flags, align_const, gen_number))
13229 can_allocate = uoh_a_fit_segment_end_p (gen_number, size,
13230 acontext, flags, align_const,
13231 commit_failed_p, oom_r);
13233 #ifdef BACKGROUND_GC
13234 if (can_allocate && gc_heap::background_running_p())
13236 if (gen_number == poh_generation)
13238 bgc_poh_size_increased += size;
13242 bgc_loh_size_increased += size;
13245 #endif //BACKGROUND_GC
13248 return can_allocate;
13251 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
13255 BOOL did_full_compact_gc = FALSE;
13257 size_t last_full_compact_gc_count = get_full_compact_gc_count();
13259 // Set this so the next GC will be a full compacting GC.
13260 if (!last_gc_before_oom)
13262 last_gc_before_oom = TRUE;
13265 #ifdef BACKGROUND_GC
13266 if (gc_heap::background_running_p())
13268 wait_for_background (((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc), loh_p);
13269 dprintf (2, ("waited for BGC - done"));
13271 #endif //BACKGROUND_GC
13273 GCSpinLock* msl = loh_p ? &more_space_lock_uoh : &more_space_lock_soh;
13274 size_t current_full_compact_gc_count = get_full_compact_gc_count();
13275 if (current_full_compact_gc_count > last_full_compact_gc_count)
13277 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
13278 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13279 did_full_compact_gc = TRUE;
13283 dprintf (3, ("h%d full GC", heap_number));
13285 trigger_gc_for_alloc (max_generation, gr, msl, loh_p, mt_t_full_gc);
13287 current_full_compact_gc_count = get_full_compact_gc_count();
13289 if (current_full_compact_gc_count == last_full_compact_gc_count)
13291 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
13292 // We requested a full GC but didn't get because of the elevation logic
13293 // which means we should fail.
13294 *oom_r = oom_unproductive_full_gc;
13298 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
13300 last_full_compact_gc_count,
13301 current_full_compact_gc_count));
13303 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13304 did_full_compact_gc = TRUE;
13308 return did_full_compact_gc;
13311 #ifdef RECORD_LOH_STATE
13312 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
13314 // When the state is can_allocate we already have released the more
13315 // space lock. So we are not logging states here since this code
13316 // is not thread safe.
13317 if (loh_state_to_save != a_state_can_allocate)
13319 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
13320 last_loh_states[loh_state_index].thread_id = thread_id;
13323 if (loh_state_index == max_saved_loh_states)
13325 loh_state_index = 0;
13328 assert (loh_state_index < max_saved_loh_states);
13331 #endif //RECORD_LOH_STATE
13333 bool gc_heap::should_retry_other_heap (int gen_number, size_t size)
13335 #ifdef MULTIPLE_HEAPS
13336 if (heap_hard_limit)
13338 size_t min_size = dd_min_size (g_heaps[0]->dynamic_data_of (gen_number));
13339 size_t slack_space = max (commit_min_th, min_size);
13340 bool retry_p = ((current_total_committed + size) < (heap_hard_limit - slack_space));
13341 dprintf (1, ("%Id - %Id - total committed %Id - size %Id = %Id, %s",
13342 heap_hard_limit, slack_space, current_total_committed, size,
13343 (heap_hard_limit - slack_space - current_total_committed - size),
13344 (retry_p ? "retry" : "no retry")));
13348 #endif //MULTIPLE_HEAPS
13354 allocation_state gc_heap::allocate_uoh (int gen_number,
13356 alloc_context* acontext,
13360 #ifdef BACKGROUND_GC
13362 if (gc_heap::background_running_p())
13364 #ifdef BGC_SERVO_TUNING
13365 bool planning_p = (current_c_gc_state == c_gc_state_planning);
13366 #endif //BGC_SERVO_TUNING
13368 background_uoh_alloc_count++;
13369 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
13371 #ifdef BGC_SERVO_TUNING
13374 loh_a_bgc_planning += size;
13378 loh_a_bgc_marking += size;
13380 #endif //BGC_SERVO_TUNING
13382 int spin_for_allocation = (gen_number == loh_generation) ?
13383 bgc_loh_allocate_spin() :
13384 bgc_poh_allocate_spin();
13386 if (spin_for_allocation > 0)
13388 add_saved_spinlock_info (true, me_release, mt_alloc_large);
13389 leave_spin_lock (&more_space_lock_uoh);
13390 bool cooperative_mode = enable_preemptive();
13391 GCToOSInterface::YieldThread (spin_for_allocation);
13392 disable_preemptive (cooperative_mode);
13393 enter_spin_lock (&more_space_lock_uoh);
13394 add_saved_spinlock_info (true, me_acquire, mt_alloc_large);
13395 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl uoh", heap_number));
13397 else if (spin_for_allocation < 0)
13399 wait_for_background (awr_uoh_alloc_during_bgc, true);
13403 #ifdef BGC_SERVO_TUNING
13406 loh_a_no_bgc += size;
13408 #endif //BGC_SERVO_TUNING
13409 #endif //BACKGROUND_GC
13411 gc_reason gr = reason_oos_loh;
13412 generation* gen = generation_of (gen_number);
13413 oom_reason oom_r = oom_no_failure;
13414 size_t current_full_compact_gc_count = 0;
13416 // No variable values should be "carried over" from one state to the other.
13417 // That's why there are local variable for each state
13418 allocation_state uoh_alloc_state = a_state_start;
13419 #ifdef RECORD_LOH_STATE
13420 EEThreadId current_thread_id;
13421 current_thread_id.SetToCurrentThread();
13422 #endif //RECORD_LOH_STATE
13424 // If we can get a new seg it means allocation will succeed.
13427 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[uoh_alloc_state]));
13429 #ifdef RECORD_LOH_STATE
13430 add_saved_loh_state (loh_uoh_alloc_state, current_thread_id);
13431 #endif //RECORD_LOH_STATE
13432 switch (uoh_alloc_state)
13434 case a_state_can_allocate:
13435 case a_state_cant_allocate:
13439 case a_state_start:
13441 uoh_alloc_state = a_state_try_fit;
13444 case a_state_try_fit:
13446 BOOL commit_failed_p = FALSE;
13447 BOOL can_use_existing_p = FALSE;
13449 can_use_existing_p = uoh_try_fit (gen_number, size, acontext, flags,
13450 align_const, &commit_failed_p, &oom_r);
13451 uoh_alloc_state = (can_use_existing_p ?
13452 a_state_can_allocate :
13454 a_state_trigger_full_compact_gc :
13455 a_state_acquire_seg));
13456 assert ((uoh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13459 case a_state_try_fit_new_seg:
13461 BOOL commit_failed_p = FALSE;
13462 BOOL can_use_existing_p = FALSE;
13464 can_use_existing_p = uoh_try_fit (gen_number, size, acontext, flags,
13465 align_const, &commit_failed_p, &oom_r);
13466 // Even after we got a new seg it doesn't necessarily mean we can allocate,
13467 // another LOH allocating thread could have beat us to acquire the msl so
13468 // we need to try again.
13469 uoh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
13470 assert ((uoh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13473 case a_state_try_fit_after_cg:
13475 BOOL commit_failed_p = FALSE;
13476 BOOL can_use_existing_p = FALSE;
13478 can_use_existing_p = uoh_try_fit (gen_number, size, acontext, flags,
13479 align_const, &commit_failed_p, &oom_r);
13480 // If we failed to commit, we bail right away 'cause we already did a
13481 // full compacting GC.
13482 uoh_alloc_state = (can_use_existing_p ?
13483 a_state_can_allocate :
13485 a_state_cant_allocate :
13486 a_state_acquire_seg_after_cg));
13487 assert ((uoh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13490 case a_state_try_fit_after_bgc:
13492 BOOL commit_failed_p = FALSE;
13493 BOOL can_use_existing_p = FALSE;
13495 can_use_existing_p = uoh_try_fit (gen_number, size, acontext, flags,
13496 align_const, &commit_failed_p, &oom_r);
13497 uoh_alloc_state = (can_use_existing_p ?
13498 a_state_can_allocate :
13500 a_state_trigger_full_compact_gc :
13501 a_state_acquire_seg_after_bgc));
13502 assert ((uoh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13505 case a_state_acquire_seg:
13507 BOOL can_get_new_seg_p = FALSE;
13508 BOOL did_full_compacting_gc = FALSE;
13510 current_full_compact_gc_count = get_full_compact_gc_count();
13512 can_get_new_seg_p = uoh_get_new_seg (gen_number, size, &did_full_compacting_gc, &oom_r);
13513 uoh_alloc_state = (can_get_new_seg_p ?
13514 a_state_try_fit_new_seg :
13515 (did_full_compacting_gc ?
13516 a_state_check_retry_seg :
13517 a_state_check_and_wait_for_bgc));
13520 case a_state_acquire_seg_after_cg:
13522 BOOL can_get_new_seg_p = FALSE;
13523 BOOL did_full_compacting_gc = FALSE;
13525 current_full_compact_gc_count = get_full_compact_gc_count();
13527 can_get_new_seg_p = uoh_get_new_seg (gen_number, size, &did_full_compacting_gc, &oom_r);
13528 // Since we release the msl before we try to allocate a seg, other
13529 // threads could have allocated a bunch of segments before us so
13530 // we might need to retry.
13531 uoh_alloc_state = (can_get_new_seg_p ?
13532 a_state_try_fit_after_cg :
13533 a_state_check_retry_seg);
13536 case a_state_acquire_seg_after_bgc:
13538 BOOL can_get_new_seg_p = FALSE;
13539 BOOL did_full_compacting_gc = FALSE;
13541 current_full_compact_gc_count = get_full_compact_gc_count();
13543 can_get_new_seg_p = uoh_get_new_seg (gen_number, size, &did_full_compacting_gc, &oom_r);
13544 uoh_alloc_state = (can_get_new_seg_p ?
13545 a_state_try_fit_new_seg :
13546 (did_full_compacting_gc ?
13547 a_state_check_retry_seg :
13548 a_state_trigger_full_compact_gc));
13549 assert ((uoh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13552 case a_state_check_and_wait_for_bgc:
13554 BOOL bgc_in_progress_p = FALSE;
13555 BOOL did_full_compacting_gc = FALSE;
13557 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc, true);
13558 uoh_alloc_state = (!bgc_in_progress_p ?
13559 a_state_trigger_full_compact_gc :
13560 (did_full_compacting_gc ?
13561 a_state_try_fit_after_cg :
13562 a_state_try_fit_after_bgc));
13565 case a_state_trigger_full_compact_gc:
13567 if (fgn_maxgen_percent)
13569 dprintf (2, ("FGN: LOH doing last GC before we throw OOM"));
13570 send_full_gc_notification (max_generation, FALSE);
13573 BOOL got_full_compacting_gc = FALSE;
13575 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, true);
13576 uoh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13577 assert ((uoh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13580 case a_state_check_retry_seg:
13582 BOOL should_retry_gc = retry_full_compact_gc (size);
13583 BOOL should_retry_get_seg = FALSE;
13584 if (!should_retry_gc)
13586 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13587 current_full_compact_gc_count = get_full_compact_gc_count();
13588 if (current_full_compact_gc_count > last_full_compact_gc_count)
13590 should_retry_get_seg = TRUE;
13594 uoh_alloc_state = (should_retry_gc ?
13595 a_state_trigger_full_compact_gc :
13596 (should_retry_get_seg ?
13597 a_state_try_fit_after_cg :
13598 a_state_cant_allocate));
13599 assert ((uoh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13604 assert (!"Invalid state!");
13611 if (uoh_alloc_state == a_state_cant_allocate)
13613 assert (oom_r != oom_no_failure);
13615 if ((oom_r != oom_cant_commit) && should_retry_other_heap (gen_number, size))
13617 uoh_alloc_state = a_state_retry_allocate;
13626 add_saved_spinlock_info (true, me_release, mt_alloc_large_cant);
13627 leave_spin_lock (&more_space_lock_uoh);
13630 assert ((uoh_alloc_state == a_state_can_allocate) ||
13631 (uoh_alloc_state == a_state_cant_allocate) ||
13632 (uoh_alloc_state == a_state_retry_allocate));
13633 return uoh_alloc_state;
13636 // BGC's final mark phase will acquire the msl, so release it here and re-acquire.
13637 void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr,
13638 GCSpinLock* msl, bool loh_p,
13639 msl_take_state take_state)
13641 #ifdef BACKGROUND_GC
13644 add_saved_spinlock_info (loh_p, me_release, take_state);
13645 leave_spin_lock (msl);
13647 #endif //BACKGROUND_GC
13649 vm_heap->GarbageCollectGeneration (gen_number, gr);
13651 #ifdef MULTIPLE_HEAPS
13654 enter_spin_lock (msl);
13655 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13657 #endif //MULTIPLE_HEAPS
13659 #ifdef BACKGROUND_GC
13662 enter_spin_lock (msl);
13663 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13665 #endif //BACKGROUND_GC
13668 allocation_state gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13669 uint32_t flags, int gen_number)
13671 if (gc_heap::gc_started)
13673 wait_for_gc_done();
13674 return a_state_retry_allocate;
13677 bool loh_p = (gen_number > 0);
13678 GCSpinLock* msl = loh_p ? &more_space_lock_uoh : &more_space_lock_soh;
13680 #ifdef SYNCHRONIZATION_STATS
13681 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13682 #endif //SYNCHRONIZATION_STATS
13683 enter_spin_lock (msl);
13684 add_saved_spinlock_info (loh_p, me_acquire, mt_try_alloc);
13685 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13686 #ifdef SYNCHRONIZATION_STATS
13687 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13688 total_msl_acquire += msl_acquire;
13689 num_msl_acquired++;
13690 if (msl_acquire > 200)
13692 num_high_msl_acquire++;
13696 num_low_msl_acquire++;
13698 #endif //SYNCHRONIZATION_STATS
13701 // We are commenting this out 'cause we don't see the point - we already
13702 // have checked gc_started when we were acquiring the msl - no need to check
13703 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13704 // need to release msl which causes all sorts of trouble.
13705 if (gc_heap::gc_started)
13707 #ifdef SYNCHRONIZATION_STATS
13709 #endif //SYNCHRONIZATION_STATS
13710 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13713 //Rendez vous early (MP scaling issue)
13714 //dprintf (1, ("[%d]waiting for gc", heap_number));
13715 wait_for_gc_done();
13716 #ifdef MULTIPLE_HEAPS
13718 #endif //MULTIPLE_HEAPS
13723 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13725 int align_const = get_alignment_constant (gen_number <= max_generation);
13727 if (fgn_maxgen_percent)
13729 check_for_full_gc (gen_number, size);
13732 #ifdef BGC_SERVO_TUNING
13733 if ((gen_number != 0) && bgc_tuning::should_trigger_bgc_loh())
13735 trigger_gc_for_alloc (max_generation, reason_bgc_tuning_loh, msl, loh_p, mt_try_servo_budget);
13738 #endif //BGC_SERVO_TUNING
13740 bool trigger_on_budget_loh_p =
13741 #ifdef BGC_SERVO_TUNING
13742 !bgc_tuning::enable_fl_tuning;
13745 #endif //BGC_SERVO_TUNING
13747 bool check_budget_p = true;
13748 if (gen_number != 0)
13750 check_budget_p = trigger_on_budget_loh_p;
13753 if (check_budget_p && !(new_allocation_allowed (gen_number)))
13755 if (fgn_maxgen_percent && (gen_number == 0))
13757 // We only check gen0 every so often, so take this opportunity to check again.
13758 check_for_full_gc (gen_number, size);
13761 #ifdef BACKGROUND_GC
13762 wait_for_bgc_high_memory (awr_gen0_alloc, loh_p);
13763 #endif //BACKGROUND_GC
13765 #ifdef SYNCHRONIZATION_STATS
13767 #endif //SYNCHRONIZATION_STATS
13768 dprintf (2, ("h%d running out of budget on gen%d, gc", heap_number, gen_number));
13770 if (!settings.concurrent || (gen_number == 0))
13772 trigger_gc_for_alloc (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh),
13773 msl, loh_p, mt_try_budget);
13778 allocation_state can_allocate = ((gen_number == 0) ?
13779 allocate_soh (gen_number, size, acontext, flags, align_const) :
13780 allocate_uoh (gen_number, size, acontext, flags, align_const));
13782 if (can_allocate == a_state_can_allocate)
13784 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13785 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13787 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13789 allocated_since_last_gc += alloc_context_bytes;
13791 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13793 #ifdef FEATURE_REDHAWK
13794 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13795 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13798 #if defined(FEATURE_EVENT_TRACE)
13799 // We are explicitly checking whether the event is enabled here.
13800 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13801 // The ones that do are much less efficient.
13802 if (EVENT_ENABLED(GCAllocationTick_V3))
13804 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13807 #endif //FEATURE_EVENT_TRACE
13808 #endif //FEATURE_REDHAWK
13809 etw_allocation_running_amount[etw_allocation_index] = 0;
13813 return can_allocate;
13816 #ifdef MULTIPLE_HEAPS
13817 void gc_heap::balance_heaps (alloc_context* acontext)
13819 if (acontext->alloc_count < 4)
13821 if (acontext->alloc_count == 0)
13823 int home_hp_num = heap_select::select_heap (acontext);
13824 acontext->set_home_heap (GCHeap::GetHeap (home_hp_num));
13825 gc_heap* hp = acontext->get_home_heap ()->pGenGCHeap;
13826 acontext->set_alloc_heap (acontext->get_home_heap ());
13827 hp->alloc_context_count++;
13829 #ifdef HEAP_BALANCE_INSTRUMENTATION
13830 uint16_t ideal_proc_no = 0;
13831 GCToOSInterface::GetCurrentThreadIdealProc (&ideal_proc_no);
13833 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber ();
13835 add_to_hb_numa (proc_no, ideal_proc_no,
13836 home_hp_num, false, true, false);
13838 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPafter GC: 1st alloc on p%3d, h%d, ip: %d",
13839 proc_no, home_hp_num, ideal_proc_no));
13840 #endif //HEAP_BALANCE_INSTRUMENTATION
13845 BOOL set_home_heap = FALSE;
13846 gc_heap* home_hp = NULL;
13847 int proc_hp_num = 0;
13849 #ifdef HEAP_BALANCE_INSTRUMENTATION
13850 bool alloc_count_p = true;
13851 bool multiple_procs_p = false;
13852 bool set_ideal_p = false;
13853 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber ();
13854 uint32_t last_proc_no = proc_no;
13855 #endif //HEAP_BALANCE_INSTRUMENTATION
13857 if (heap_select::can_find_heap_fast ())
13859 assert (acontext->get_home_heap () != NULL);
13860 home_hp = acontext->get_home_heap ()->pGenGCHeap;
13861 proc_hp_num = heap_select::select_heap (acontext);
13863 if (acontext->get_home_heap () != GCHeap::GetHeap (proc_hp_num))
13865 #ifdef HEAP_BALANCE_INSTRUMENTATION
13866 alloc_count_p = false;
13867 #endif //HEAP_BALANCE_INSTRUMENTATION
13868 set_home_heap = TRUE;
13870 else if ((acontext->alloc_count & 15) == 0)
13871 set_home_heap = TRUE;
13879 if ((acontext->alloc_count & 3) == 0)
13880 set_home_heap = TRUE;
13886 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13887 if (n_heaps > MAX_SUPPORTED_CPUS)
13889 // on machines with many processors cache affinity is really king, so don't even try
13890 // to balance on these.
13891 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext));
13892 acontext->alloc_heap = acontext->home_heap;
13897 gc_heap* org_hp = acontext->get_alloc_heap ()->pGenGCHeap;
13898 int org_hp_num = org_hp->heap_number;
13899 int final_alloc_hp_num = org_hp_num;
13901 dynamic_data* dd = org_hp->dynamic_data_of (0);
13902 ptrdiff_t org_size = dd_new_allocation (dd);
13903 ptrdiff_t total_size = (ptrdiff_t)dd_desired_allocation (dd);
13905 #ifdef HEAP_BALANCE_INSTRUMENTATION
13906 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMP[p%3d] ph h%3d, hh: %3d, ah: %3d (%dmb-%dmb), ac: %5d(%s)",
13907 proc_no, proc_hp_num, home_hp->heap_number,
13908 org_hp_num, (total_size / 1024 / 1024), (org_size / 1024 / 1024),
13909 acontext->alloc_count,
13910 ((proc_hp_num == home_hp->heap_number) ? "AC" : "H")));
13911 #endif //HEAP_BALANCE_INSTRUMENTATION
13913 int org_alloc_context_count;
13914 int max_alloc_context_count;
13916 int max_hp_num = 0;
13917 ptrdiff_t max_size;
13918 size_t local_delta = max (((size_t)org_size >> 6), min_gen0_balance_delta);
13919 size_t delta = local_delta;
13921 if (((size_t)org_size + 2 * delta) >= (size_t)total_size)
13923 acontext->alloc_count++;
13927 int start, end, finish;
13928 heap_select::get_heap_range_for_heap (org_hp->heap_number, &start, &end);
13929 finish = start + n_heaps;
13932 gc_heap* new_home_hp = 0;
13937 max_hp_num = org_hp_num;
13938 max_size = org_size + delta;
13939 #ifdef HEAP_BALANCE_INSTRUMENTATION
13940 proc_no = GCToOSInterface::GetCurrentProcessorNumber ();
13941 if (proc_no != last_proc_no)
13943 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPSP: %d->%d", last_proc_no, proc_no));
13944 multiple_procs_p = true;
13945 last_proc_no = proc_no;
13948 int current_hp_num = heap_select::proc_no_to_heap_no[proc_no];
13949 acontext->set_home_heap (GCHeap::GetHeap (current_hp_num));
13951 acontext->set_home_heap (GCHeap::GetHeap (heap_select::select_heap (acontext)));
13952 #endif //HEAP_BALANCE_INSTRUMENTATION
13953 new_home_hp = acontext->get_home_heap ()->pGenGCHeap;
13954 if (org_hp == new_home_hp)
13955 max_size = max_size + delta;
13957 org_alloc_context_count = org_hp->alloc_context_count;
13958 max_alloc_context_count = org_alloc_context_count;
13959 if (max_alloc_context_count > 1)
13960 max_size /= max_alloc_context_count;
13962 int actual_start = start;
13963 int actual_end = (end - 1);
13965 for (int i = start; i < end; i++)
13967 gc_heap* hp = GCHeap::GetHeap (i % n_heaps)->pGenGCHeap;
13968 dd = hp->dynamic_data_of (0);
13969 ptrdiff_t size = dd_new_allocation (dd);
13971 if (hp == new_home_hp)
13973 size = size + delta;
13975 int hp_alloc_context_count = hp->alloc_context_count;
13977 if (hp_alloc_context_count > 0)
13979 size /= (hp_alloc_context_count + 1);
13981 if (size > max_size)
13983 #ifdef HEAP_BALANCE_INSTRUMENTATION
13984 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPorg h%d(%dmb), m h%d(%dmb)",
13985 org_hp_num, (max_size / 1024 / 1024),
13986 hp->heap_number, (size / 1024 / 1024)));
13987 #endif //HEAP_BALANCE_INSTRUMENTATION
13991 max_hp_num = max_hp->heap_number;
13992 max_alloc_context_count = hp_alloc_context_count;
13996 while (org_alloc_context_count != org_hp->alloc_context_count ||
13997 max_alloc_context_count != max_hp->alloc_context_count);
13999 if ((max_hp == org_hp) && (end < finish))
14001 start = end; end = finish;
14002 delta = local_delta * 2; // Make it twice as hard to balance to remote nodes on NUMA.
14006 #ifdef HEAP_BALANCE_INSTRUMENTATION
14007 uint16_t ideal_proc_no_before_set_ideal = 0;
14008 GCToOSInterface::GetCurrentThreadIdealProc (&ideal_proc_no_before_set_ideal);
14009 #endif //HEAP_BALANCE_INSTRUMENTATION
14011 if (max_hp != org_hp)
14013 final_alloc_hp_num = max_hp->heap_number;
14015 org_hp->alloc_context_count--;
14016 max_hp->alloc_context_count++;
14018 acontext->set_alloc_heap (GCHeap::GetHeap (final_alloc_hp_num));
14019 if (!gc_thread_no_affinitize_p)
14021 uint16_t src_proc_no = heap_select::find_proc_no_from_heap_no (org_hp->heap_number);
14022 uint16_t dst_proc_no = heap_select::find_proc_no_from_heap_no (max_hp->heap_number);
14024 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPSW! h%d(p%d)->h%d(p%d)",
14025 org_hp_num, src_proc_no, final_alloc_hp_num, dst_proc_no));
14027 #ifdef HEAP_BALANCE_INSTRUMENTATION
14028 int current_proc_no_before_set_ideal = GCToOSInterface::GetCurrentProcessorNumber ();
14029 if (current_proc_no_before_set_ideal != last_proc_no)
14031 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPSPa: %d->%d", last_proc_no, current_proc_no_before_set_ideal));
14032 multiple_procs_p = true;
14034 #endif //HEAP_BALANCE_INSTRUMENTATION
14036 if (!GCToOSInterface::SetCurrentThreadIdealAffinity (src_proc_no, dst_proc_no))
14038 dprintf (HEAP_BALANCE_TEMP_LOG, ("TEMPFailed to set the ideal processor for heap %d %d->%d",
14039 org_hp->heap_number, (int)src_proc_no, (int)dst_proc_no));
14041 #ifdef HEAP_BALANCE_INSTRUMENTATION
14044 set_ideal_p = true;
14046 #endif //HEAP_BALANCE_INSTRUMENTATION
14050 #ifdef HEAP_BALANCE_INSTRUMENTATION
14051 add_to_hb_numa (proc_no, ideal_proc_no_before_set_ideal,
14052 final_alloc_hp_num, multiple_procs_p, alloc_count_p, set_ideal_p);
14053 #endif //HEAP_BALANCE_INSTRUMENTATION
14057 acontext->alloc_count++;
14060 ptrdiff_t gc_heap::get_balance_heaps_uoh_effective_budget (int generation_num)
14062 if (heap_hard_limit)
14064 const ptrdiff_t free_list_space = generation_free_list_space (generation_of (generation_num));
14065 heap_segment* seg = generation_start_segment (generation_of (generation_num));
14066 assert (heap_segment_next (seg) == nullptr);
14067 const ptrdiff_t allocated = heap_segment_allocated (seg) - seg->mem;
14068 // We could calculate the actual end_of_seg_space by taking reserved - allocated,
14069 // but all heaps have the same reserved memory and this value is only used for comparison.
14070 return free_list_space - allocated;
14074 return dd_new_allocation (dynamic_data_of (generation_num));
14078 gc_heap* gc_heap::balance_heaps_uoh (alloc_context* acontext, size_t alloc_size, int generation_num)
14080 const int home_hp_num = heap_select::select_heap(acontext);
14081 dprintf (3, ("[h%d] LA: %Id", home_hp_num, alloc_size));
14082 gc_heap* home_hp = GCHeap::GetHeap(home_hp_num)->pGenGCHeap;
14083 dynamic_data* dd = home_hp->dynamic_data_of (generation_num);
14084 const ptrdiff_t home_hp_size = home_hp->get_balance_heaps_uoh_effective_budget (generation_num);
14086 size_t delta = dd_min_size (dd) / 2;
14088 heap_select::get_heap_range_for_heap(home_hp_num, &start, &end);
14089 const int finish = start + n_heaps;
14092 gc_heap* max_hp = home_hp;
14093 ptrdiff_t max_size = home_hp_size + delta;
14095 dprintf (3, ("home hp: %d, max size: %d",
14099 for (int i = start; i < end; i++)
14101 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
14102 const ptrdiff_t size = hp->get_balance_heaps_uoh_effective_budget (generation_num);
14104 dprintf (3, ("hp: %d, size: %d", hp->heap_number, size));
14105 if (size > max_size)
14109 dprintf (3, ("max hp: %d, max size: %d",
14110 max_hp->heap_number,
14115 if ((max_hp == home_hp) && (end < finish))
14117 start = end; end = finish;
14118 delta = dd_min_size (dd) * 3 / 2; // Make it harder to balance to remote nodes on NUMA.
14122 if (max_hp != home_hp)
14124 dprintf (3, ("uoh: %d(%Id)->%d(%Id)",
14125 home_hp->heap_number, dd_new_allocation (home_hp->dynamic_data_of (generation_num)),
14126 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (generation_num))));
14132 gc_heap* gc_heap::balance_heaps_uoh_hard_limit_retry (alloc_context* acontext, size_t alloc_size, int generation_num)
14134 assert (heap_hard_limit);
14135 const int home_heap = heap_select::select_heap(acontext);
14136 dprintf (3, ("[h%d] balance_heaps_loh_hard_limit_retry alloc_size: %d", home_heap, alloc_size));
14138 heap_select::get_heap_range_for_heap (home_heap, &start, &end);
14139 const int finish = start + n_heaps;
14141 gc_heap* max_hp = nullptr;
14142 size_t max_end_of_seg_space = alloc_size; // Must be more than this much, or return NULL
14146 for (int i = start; i < end; i++)
14148 gc_heap* hp = GCHeap::GetHeap (i%n_heaps)->pGenGCHeap;
14149 heap_segment* seg = generation_start_segment (hp->generation_of (generation_num));
14150 // With a hard limit, there is only one segment.
14151 assert (heap_segment_next (seg) == nullptr);
14152 const size_t end_of_seg_space = heap_segment_reserved (seg) - heap_segment_allocated (seg);
14153 if (end_of_seg_space >= max_end_of_seg_space)
14155 dprintf (3, ("Switching heaps in hard_limit_retry! To: [h%d], New end_of_seg_space: %d", hp->heap_number, end_of_seg_space));
14156 max_end_of_seg_space = end_of_seg_space;
14162 // Only switch to a remote NUMA node if we didn't find space on this one.
14163 if ((max_hp == nullptr) && (end < finish))
14165 start = end; end = finish;
14171 #endif //MULTIPLE_HEAPS
14173 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
14174 uint32_t flags, int alloc_generation_number)
14176 allocation_state status = a_state_start;
14179 #ifdef MULTIPLE_HEAPS
14180 if (alloc_generation_number == 0)
14182 balance_heaps (acontext);
14183 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, flags, alloc_generation_number);
14187 gc_heap* alloc_heap;
14188 if (heap_hard_limit && (status == a_state_retry_allocate))
14190 alloc_heap = balance_heaps_uoh_hard_limit_retry (acontext, size, alloc_generation_number);
14191 if (alloc_heap == nullptr)
14198 alloc_heap = balance_heaps_uoh (acontext, size, alloc_generation_number);
14201 status = alloc_heap->try_allocate_more_space (acontext, size, flags, alloc_generation_number);
14202 if (status == a_state_retry_allocate)
14204 dprintf (3, ("UOH h%d alloc retry!", alloc_heap->heap_number));
14208 status = try_allocate_more_space (acontext, size, flags, alloc_generation_number);
14209 #endif //MULTIPLE_HEAPS
14211 while (status == a_state_retry_allocate);
14213 return (status == a_state_can_allocate);
14217 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext, uint32_t flags)
14219 size_t size = Align (jsize);
14220 assert (size >= Align (min_obj_size));
14223 uint8_t* result = acontext->alloc_ptr;
14224 acontext->alloc_ptr+=size;
14225 if (acontext->alloc_ptr <= acontext->alloc_limit)
14227 CObjectHeader* obj = (CObjectHeader*)result;
14233 acontext->alloc_ptr -= size;
14236 #pragma inline_depth(0)
14239 if (! allocate_more_space (acontext, size, flags, 0))
14243 #pragma inline_depth(20)
14251 void gc_heap::leave_allocation_segment (generation* gen)
14253 adjust_limit (0, 0, gen);
14256 void gc_heap::init_free_and_plug()
14258 #ifdef FREE_USAGE_STATS
14259 for (int i = 0; i <= settings.condemned_generation; i++)
14261 generation* gen = generation_of (i);
14262 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
14263 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
14264 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
14267 if (settings.condemned_generation != max_generation)
14269 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
14271 generation* gen = generation_of (i);
14272 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
14275 #endif //FREE_USAGE_STATS
14278 void gc_heap::print_free_and_plug (const char* msg)
14280 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
14281 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
14282 for (int i = 0; i <= older_gen; i++)
14284 generation* gen = generation_of (i);
14285 for (int j = 0; j < NUM_GEN_POWER2; j++)
14287 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
14289 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
14292 (settings.concurrent ? "BGC" : "GC"),
14295 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
14300 UNREFERENCED_PARAMETER(msg);
14301 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
14304 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
14306 #ifdef FREE_USAGE_STATS
14307 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
14308 generation* gen = generation_of (gen_number);
14309 size_t sz = BASE_GEN_SIZE;
14312 for (; i < NUM_GEN_POWER2; i++)
14314 if (plug_size < sz)
14321 (gen->gen_plugs[i])++;
14323 UNREFERENCED_PARAMETER(gen_number);
14324 UNREFERENCED_PARAMETER(plug_size);
14325 #endif //FREE_USAGE_STATS
14328 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
14330 #ifdef FREE_USAGE_STATS
14331 generation* gen = generation_of (gen_number);
14332 size_t sz = BASE_GEN_SIZE;
14335 for (; i < NUM_GEN_POWER2; i++)
14337 if (free_size < sz)
14344 (gen->gen_current_pinned_free_spaces[i])++;
14345 generation_pinned_free_obj_space (gen) += free_size;
14346 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
14347 free_size, (i + 10), gen_number,
14348 generation_pinned_free_obj_space (gen),
14349 gen->gen_current_pinned_free_spaces[i]));
14351 UNREFERENCED_PARAMETER(gen_number);
14352 UNREFERENCED_PARAMETER(free_size);
14353 #endif //FREE_USAGE_STATS
14356 void gc_heap::add_gen_free (int gen_number, size_t free_size)
14358 #ifdef FREE_USAGE_STATS
14359 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
14360 generation* gen = generation_of (gen_number);
14361 size_t sz = BASE_GEN_SIZE;
14364 for (; i < NUM_GEN_POWER2; i++)
14366 if (free_size < sz)
14373 (gen->gen_free_spaces[i])++;
14375 UNREFERENCED_PARAMETER(gen_number);
14376 UNREFERENCED_PARAMETER(free_size);
14377 #endif //FREE_USAGE_STATS
14380 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
14382 #ifdef FREE_USAGE_STATS
14383 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
14384 generation* gen = generation_of (gen_number);
14385 size_t sz = BASE_GEN_SIZE;
14388 for (; i < NUM_GEN_POWER2; i++)
14390 if (free_size < sz)
14397 (gen->gen_free_spaces[i])--;
14399 UNREFERENCED_PARAMETER(gen_number);
14400 UNREFERENCED_PARAMETER(free_size);
14401 #endif //FREE_USAGE_STATS
14404 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
14405 int from_gen_number,
14406 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
14408 size = Align (size);
14409 assert (size >= Align (min_obj_size));
14410 assert (from_gen_number < max_generation);
14411 assert (from_gen_number >= 0);
14412 assert (generation_of (from_gen_number + 1) == gen);
14414 allocator* gen_allocator = generation_allocator (gen);
14415 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
14417 int pad_in_front = ((old_loc != 0) && ((from_gen_number+1) != max_generation)) ? USE_PADDING_FRONT : 0;
14418 #else //SHORT_PLUGS
14419 int pad_in_front = 0;
14420 #endif //SHORT_PLUGS
14422 size_t real_size = size + Align (min_obj_size);
14424 real_size += Align (min_obj_size);
14426 #ifdef RESPECT_LARGE_ALIGNMENT
14427 real_size += switch_alignment_size (pad_in_front);
14428 #endif //RESPECT_LARGE_ALIGNMENT
14430 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14431 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
14433 for (unsigned int a_l_idx = gen_allocator->first_suitable_bucket(real_size * 2); a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
14435 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
14436 uint8_t* prev_free_item = 0;
14437 while (free_list != 0)
14439 dprintf (3, ("considering free list %Ix", (size_t)free_list));
14441 size_t free_list_size = unused_array_size (free_list);
14443 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
14444 old_loc, USE_PADDING_TAIL | pad_in_front))
14446 dprintf (4, ("F:%Ix-%Id",
14447 (size_t)free_list, free_list_size));
14449 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
14450 generation_free_list_space (gen) -= free_list_size;
14451 remove_gen_free (gen->gen_num, free_list_size);
14453 adjust_limit (free_list, free_list_size, gen);
14454 generation_allocate_end_seg_p (gen) = FALSE;
14457 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
14458 else if (discard_p || (a_l_idx == 0))
14460 dprintf (3, ("couldn't use this free area, discarding"));
14461 generation_free_obj_space (gen) += free_list_size;
14463 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
14464 generation_free_list_space (gen) -= free_list_size;
14465 remove_gen_free (gen->gen_num, free_list_size);
14469 prev_free_item = free_list;
14471 free_list = free_list_slot (free_list);
14474 //go back to the beginning of the segment list
14475 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
14476 if (seg != generation_allocation_segment (gen))
14478 leave_allocation_segment (gen);
14479 generation_allocation_segment (gen) = seg;
14481 while (seg != ephemeral_heap_segment)
14483 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14484 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
14486 dprintf (3, ("using what's left in committed"));
14487 adjust_limit (heap_segment_plan_allocated (seg),
14488 (heap_segment_committed (seg) - heap_segment_plan_allocated (seg)),
14490 generation_allocate_end_seg_p (gen) = TRUE;
14491 // dformat (t, 3, "Expanding segment allocation");
14492 heap_segment_plan_allocated (seg) =
14493 heap_segment_committed (seg);
14498 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14499 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14500 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
14502 dprintf (3, ("using what's left in reserved"));
14503 adjust_limit (heap_segment_plan_allocated (seg),
14504 (heap_segment_committed (seg) - heap_segment_plan_allocated (seg)),
14506 generation_allocate_end_seg_p (gen) = TRUE;
14507 heap_segment_plan_allocated (seg) =
14508 heap_segment_committed (seg);
14514 leave_allocation_segment (gen);
14515 heap_segment* next_seg = heap_segment_next_rw (seg);
14518 dprintf (3, ("getting next segment"));
14519 generation_allocation_segment (gen) = next_seg;
14520 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14521 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14530 seg = generation_allocation_segment (gen);
14532 //No need to fix the last region. Will be done later
14543 uint8_t* result = generation_allocation_pointer (gen);
14547 if ((pad_in_front & USE_PADDING_FRONT) &&
14548 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14549 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14551 pad = Align (min_obj_size);
14552 set_plug_padded (old_loc);
14554 #endif //SHORT_PLUGS
14556 #ifdef FEATURE_STRUCTALIGN
14557 _ASSERTE(!old_loc || alignmentOffset != 0);
14558 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14561 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14562 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14565 #else // FEATURE_STRUCTALIGN
14566 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14568 pad += switch_alignment_size (pad != 0);
14569 set_node_realigned (old_loc);
14570 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14571 (size_t)old_loc, (size_t)(result+pad)));
14572 assert (same_large_alignment_p (result + pad, old_loc));
14574 #endif // FEATURE_STRUCTALIGN
14575 dprintf (3, ("Allocate %Id bytes", size));
14577 if ((old_loc == 0) || (pad != 0))
14579 //allocating a non plug or a gap, so reset the start region
14580 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14583 generation_allocation_pointer (gen) += size + pad;
14584 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14585 if (generation_allocate_end_seg_p (gen))
14587 generation_end_seg_allocated (gen) += size;
14591 generation_free_list_allocated (gen) += size;
14593 generation_allocation_size (gen) += size;
14595 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
14596 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14597 generation_allocation_context_start_region (gen)));
14599 return result + pad;;
14603 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
14605 //make sure that every generation has a planned allocation start
14606 int gen_number = max_generation - 1;
14607 while (gen_number>= 0)
14609 generation* gen = generation_of (gen_number);
14610 if (0 == generation_plan_allocation_start (gen))
14612 realloc_plan_generation_start (gen, consing_gen);
14614 assert (generation_plan_allocation_start (gen));
14619 // now we know the planned allocation size
14620 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
14621 heap_segment* seg = generation_allocation_segment (consing_gen);
14622 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
14626 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14631 assert (settings.condemned_generation == max_generation);
14632 uint8_t* first_address = generation_allocation_limit (consing_gen);
14633 //look through the pinned plugs for relevant ones.
14634 //Look for the right pinned plug to start from.
14637 while (mi != mark_stack_tos)
14639 m = pinned_plug_of (mi);
14640 if ((pinned_plug (m) == first_address))
14645 assert (mi != mark_stack_tos);
14646 pinned_len (m) = size;
14650 //tododefrag optimize for new segment (plan_allocated == mem)
14651 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
14656 BOOL set_padding_on_saved_p,
14657 mark* pinned_plug_entry,
14658 #endif //SHORT_PLUGS
14659 BOOL consider_bestfit,
14660 int active_new_gen_number
14661 REQD_ALIGN_AND_OFFSET_DCL)
14663 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
14665 size = Align (size);
14666 assert (size >= Align (min_obj_size));
14668 int pad_in_front = ((old_loc != 0) && (active_new_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14669 #else //SHORT_PLUGS
14670 int pad_in_front = 0;
14671 #endif //SHORT_PLUGS
14673 if (consider_bestfit && use_bestfit)
14675 assert (bestfit_seg);
14676 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
14678 return bestfit_seg->fit (old_loc,
14679 size REQD_ALIGN_AND_OFFSET_ARG);
14682 heap_segment* seg = generation_allocation_segment (gen);
14684 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14685 generation_allocation_limit (gen), old_loc,
14686 ((generation_allocation_limit (gen) !=
14687 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
14689 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
14690 generation_allocation_limit (gen)));
14693 uint8_t* first_address = (generation_allocation_limit (gen) ?
14694 generation_allocation_limit (gen) :
14695 heap_segment_mem (seg));
14696 assert (in_range_for_segment (first_address, seg));
14698 uint8_t* end_address = heap_segment_reserved (seg);
14700 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
14701 first_address, generation_allocation_limit (gen), end_address));
14706 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
14708 assert (settings.condemned_generation == max_generation);
14709 //look through the pinned plugs for relevant ones.
14710 //Look for the right pinned plug to start from.
14711 while (mi != mark_stack_tos)
14713 m = pinned_plug_of (mi);
14714 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14716 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14722 if (mi != mark_stack_tos)
14724 //fix old free list.
14725 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14727 dprintf(3,("gc filling up hole"));
14728 ptrdiff_t mi1 = (ptrdiff_t)mi;
14729 while ((mi1 >= 0) &&
14730 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14732 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14737 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14738 pinned_len (pinned_plug_of(mi1)) = hsize;
14739 dprintf (3, ("changing %Ix len %Ix->%Ix",
14740 pinned_plug (pinned_plug_of(mi1)),
14741 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14748 assert (generation_allocation_limit (gen) ==
14749 generation_allocation_pointer (gen));
14750 mi = mark_stack_tos;
14753 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14755 size_t len = pinned_len (m);
14756 uint8_t* free_list = (pinned_plug (m) - len);
14757 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14758 free_list, (free_list + len), len));
14759 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14761 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14762 (size_t)free_list, len));
14764 generation_allocation_pointer (gen) = free_list;
14765 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14766 generation_allocation_limit (gen) = (free_list + len);
14768 goto allocate_in_free;
14771 m = pinned_plug_of (mi);
14774 //switch to the end of the segment.
14775 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14776 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14777 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14778 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14779 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14780 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14781 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14783 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14784 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14786 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14787 generation_allocation_limit (gen)));
14788 assert (!"Can't allocate if no free space");
14799 uint8_t* result = generation_allocation_pointer (gen);
14803 if ((pad_in_front & USE_PADDING_FRONT) &&
14804 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14805 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14808 pad = Align (min_obj_size);
14809 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14811 #endif //SHORT_PLUGS
14813 #ifdef FEATURE_STRUCTALIGN
14814 _ASSERTE(!old_loc || alignmentOffset != 0);
14815 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14818 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14819 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14823 #else // FEATURE_STRUCTALIGN
14824 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14826 pad += switch_alignment_size (pad != 0);
14827 set_node_realigned (old_loc);
14828 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14829 (size_t)old_loc, (size_t)(result+pad)));
14830 assert (same_large_alignment_p (result + pad, old_loc));
14833 #endif // FEATURE_STRUCTALIGN
14835 if ((old_loc == 0) || (pad != 0))
14837 //allocating a non plug or a gap, so reset the start region
14838 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14841 generation_allocation_pointer (gen) += size + pad;
14842 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14843 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14845 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14846 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14847 generation_allocation_context_start_region (gen)));
14849 return result + pad;
14853 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14855 heap_segment* seg = generation_allocation_segment (consing_gen);
14856 if (seg != ephemeral_heap_segment)
14858 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14859 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14861 //fix the allocated size of the segment.
14862 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14864 generation* new_consing_gen = generation_of (max_generation - 1);
14865 generation_allocation_pointer (new_consing_gen) =
14866 heap_segment_mem (ephemeral_heap_segment);
14867 generation_allocation_limit (new_consing_gen) =
14868 generation_allocation_pointer (new_consing_gen);
14869 generation_allocation_context_start_region (new_consing_gen) =
14870 generation_allocation_pointer (new_consing_gen);
14871 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14873 return new_consing_gen;
14876 return consing_gen;
14879 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14881 int from_gen_number,
14883 BOOL* convert_to_pinned_p,
14884 uint8_t* next_pinned_plug,
14885 heap_segment* current_seg,
14886 #endif //SHORT_PLUGS
14888 REQD_ALIGN_AND_OFFSET_DCL)
14890 // Make sure that the youngest generation gap hasn't been allocated
14891 if (settings.promotion)
14893 assert (generation_plan_allocation_start (youngest_generation) == 0);
14896 size = Align (size);
14897 assert (size >= Align (min_obj_size));
14898 int to_gen_number = from_gen_number;
14899 if (from_gen_number != (int)max_generation)
14901 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14904 dprintf (3, ("aic gen%d: s: %Id", gen->gen_num, size));
14907 int pad_in_front = ((old_loc != 0) && (to_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14908 #else //SHORT_PLUGS
14909 int pad_in_front = 0;
14910 #endif //SHORT_PLUGS
14912 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14914 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14915 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14919 heap_segment* seg = generation_allocation_segment (gen);
14920 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14921 generation_allocation_limit (gen), old_loc,
14922 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14924 if ((! (pinned_plug_que_empty_p()) &&
14925 (generation_allocation_limit (gen) ==
14926 pinned_plug (oldest_pin()))))
14928 size_t entry = deque_pinned_plug();
14929 mark* pinned_plug_entry = pinned_plug_of (entry);
14930 size_t len = pinned_len (pinned_plug_entry);
14931 uint8_t* plug = pinned_plug (pinned_plug_entry);
14932 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14934 #ifdef FREE_USAGE_STATS
14935 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14936 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14937 generation_allocated_since_last_pin (gen),
14939 generation_allocated_in_pinned_free (gen)));
14940 generation_allocated_since_last_pin (gen) = 0;
14942 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14943 #endif //FREE_USAGE_STATS
14945 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14946 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14948 assert(mark_stack_array[entry].len == 0 ||
14949 mark_stack_array[entry].len >= Align(min_obj_size));
14950 generation_allocation_pointer (gen) = plug + len;
14951 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14952 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14953 set_allocator_next_pin (gen);
14955 //Add the size of the pinned plug to the right pinned allocations
14956 //find out which gen this pinned plug came from
14957 int frgn = object_gennum (plug);
14958 if ((frgn != (int)max_generation) && settings.promotion)
14960 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14961 int togn = object_gennum_plan (plug);
14964 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14970 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14972 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14973 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14977 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14979 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14980 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14981 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14985 #ifndef RESPECT_LARGE_ALIGNMENT
14986 assert (gen != youngest_generation);
14987 #endif //RESPECT_LARGE_ALIGNMENT
14989 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14990 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14991 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14992 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14994 dprintf (3, ("Expanded segment allocation by committing more memory"));
14995 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14996 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
15000 heap_segment* next_seg = heap_segment_next (seg);
15001 assert (generation_allocation_pointer (gen)>=
15002 heap_segment_mem (seg));
15003 // Verify that all pinned plugs for this segment are consumed
15004 if (!pinned_plug_que_empty_p() &&
15005 ((pinned_plug (oldest_pin()) <
15006 heap_segment_allocated (seg)) &&
15007 (pinned_plug (oldest_pin()) >=
15008 generation_allocation_pointer (gen))))
15010 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
15011 pinned_plug (oldest_pin())));
15014 assert (generation_allocation_pointer (gen)>=
15015 heap_segment_mem (seg));
15016 assert (generation_allocation_pointer (gen)<=
15017 heap_segment_committed (seg));
15018 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
15022 generation_allocation_segment (gen) = next_seg;
15023 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
15024 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
15025 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
15029 return 0; //should only happen during allocation of generation 0 gap
15030 // in that case we are going to grow the heap anyway
15035 set_allocator_next_pin (gen);
15042 assert (generation_allocation_pointer (gen)>=
15043 heap_segment_mem (generation_allocation_segment (gen)));
15044 uint8_t* result = generation_allocation_pointer (gen);
15047 if ((pad_in_front & USE_PADDING_FRONT) &&
15048 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
15049 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
15051 ptrdiff_t dist = old_loc - result;
15054 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
15059 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
15061 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
15065 pad = Align (min_obj_size);
15066 set_plug_padded (old_loc);
15069 #endif //SHORT_PLUGS
15070 #ifdef FEATURE_STRUCTALIGN
15071 _ASSERTE(!old_loc || alignmentOffset != 0);
15072 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
15073 if ((old_loc != 0))
15075 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
15076 set_node_aligninfo (old_loc, requiredAlignment, pad1);
15079 #else // FEATURE_STRUCTALIGN
15080 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
15082 pad += switch_alignment_size (pad != 0);
15083 set_node_realigned(old_loc);
15084 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
15085 (size_t)old_loc, (size_t)(result+pad)));
15086 assert (same_large_alignment_p (result + pad, old_loc));
15088 #endif // FEATURE_STRUCTALIGN
15091 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
15093 assert (old_loc != 0);
15094 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
15095 assert (dist_to_next_pin >= 0);
15097 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
15099 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
15101 generation_allocation_pointer (gen),
15102 generation_allocation_limit (gen),
15105 dist_to_next_pin));
15106 clear_plug_padded (old_loc);
15108 *convert_to_pinned_p = TRUE;
15109 record_interesting_data_point (idp_converted_pin);
15114 #endif //SHORT_PLUGS
15116 if ((old_loc == 0) || (pad != 0))
15118 //allocating a non plug or a gap, so reset the start region
15119 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
15122 generation_allocation_pointer (gen) += size + pad;
15123 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
15125 #ifdef FREE_USAGE_STATS
15126 generation_allocated_since_last_pin (gen) += size;
15127 #endif //FREE_USAGE_STATS
15129 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
15130 generation_allocation_pointer (gen), generation_allocation_limit (gen),
15131 generation_allocation_context_start_region (gen)));
15133 assert (result + pad);
15134 return result + pad;
15138 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
15141 BOOL* blocking_collection_p
15142 STRESS_HEAP_ARG(int n_original))
15144 gc_data_global.gen_to_condemn_reasons.init();
15145 #ifdef BGC_SERVO_TUNING
15146 if (settings.entry_memory_load == 0)
15148 uint32_t current_memory_load = 0;
15149 uint64_t current_available_physical = 0;
15150 get_memory_info (¤t_memory_load, ¤t_available_physical);
15152 settings.entry_memory_load = current_memory_load;
15153 settings.entry_available_physical_mem = current_available_physical;
15155 #endif //BGC_SERVO_TUNING
15157 int n = current_gen;
15158 #ifdef MULTIPLE_HEAPS
15159 BOOL joined_last_gc_before_oom = FALSE;
15160 for (int i = 0; i < n_heaps; i++)
15162 if (g_heaps[i]->last_gc_before_oom)
15164 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
15165 joined_last_gc_before_oom = TRUE;
15170 BOOL joined_last_gc_before_oom = last_gc_before_oom;
15171 #endif //MULTIPLE_HEAPS
15173 if (joined_last_gc_before_oom && settings.pause_mode != pause_low_latency)
15175 assert (*blocking_collection_p);
15178 if (should_evaluate_elevation && (n == max_generation))
15180 dprintf (GTC_LOG, ("lock: %d(%d)",
15181 (settings.should_lock_elevation ? 1 : 0),
15182 settings.elevation_locked_count));
15184 if (settings.should_lock_elevation)
15186 settings.elevation_locked_count++;
15187 if (settings.elevation_locked_count == 6)
15189 settings.elevation_locked_count = 0;
15193 n = max_generation - 1;
15194 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_avoid_unproductive);
15195 settings.elevation_reduced = TRUE;
15200 settings.elevation_locked_count = 0;
15205 settings.should_lock_elevation = FALSE;
15206 settings.elevation_locked_count = 0;
15209 if (provisional_mode_triggered && (n == max_generation))
15211 // There are a few cases where we should not reduce the generation.
15212 if ((initial_gen == max_generation) || (settings.reason == reason_alloc_loh))
15214 // If we are doing a full GC in the provisional mode, we always
15215 // make it blocking because we don't want to get into a situation
15216 // where foreground GCs are asking for a compacting full GC right away
15217 // and not getting it.
15218 dprintf (GTC_LOG, ("full GC induced, not reducing gen"));
15219 if (initial_gen == max_generation)
15221 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_pm_induced_fullgc_p);
15225 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_pm_alloc_loh);
15227 *blocking_collection_p = TRUE;
15229 else if (should_expand_in_full_gc || joined_last_gc_before_oom)
15231 dprintf (GTC_LOG, ("need full blocking GCs to expand heap or avoid OOM, not reducing gen"));
15232 assert (*blocking_collection_p);
15236 dprintf (GTC_LOG, ("reducing gen in PM: %d->%d->%d", initial_gen, n, (max_generation - 1)));
15237 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_gen1_in_pm);
15238 n = max_generation - 1;
15242 if (should_expand_in_full_gc)
15244 should_expand_in_full_gc = FALSE;
15247 if (heap_hard_limit)
15249 // If we have already consumed 90% of the limit, we should check to see if we should compact LOH.
15250 // TODO: should unify this with gen2.
15251 dprintf (GTC_LOG, ("committed %Id is %d%% of limit %Id",
15252 current_total_committed, (int)((float)current_total_committed * 100.0 / (float)heap_hard_limit),
15255 bool full_compact_gc_p = false;
15257 if (joined_last_gc_before_oom)
15259 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_limit_before_oom);
15260 full_compact_gc_p = true;
15262 else if ((current_total_committed * 10) >= (heap_hard_limit * 9))
15264 size_t loh_frag = get_total_gen_fragmentation (loh_generation);
15266 // If the LOH frag is >= 1/8 it's worth compacting it
15267 if ((loh_frag * 8) >= heap_hard_limit)
15269 dprintf (GTC_LOG, ("loh frag: %Id > 1/8 of limit %Id", loh_frag, (heap_hard_limit / 8)));
15270 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_limit_loh_frag);
15271 full_compact_gc_p = true;
15275 // If there's not much fragmentation but it looks like it'll be productive to
15276 // collect LOH, do that.
15277 size_t est_loh_reclaim = get_total_gen_estimated_reclaim (loh_generation);
15278 if ((est_loh_reclaim * 8) >= heap_hard_limit)
15280 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_limit_loh_reclaim);
15281 full_compact_gc_p = true;
15283 dprintf (GTC_LOG, ("loh est reclaim: %Id, 1/8 of limit %Id", est_loh_reclaim, (heap_hard_limit / 8)));
15287 if (full_compact_gc_p)
15289 n = max_generation;
15290 *blocking_collection_p = TRUE;
15291 settings.loh_compaction = TRUE;
15292 dprintf (GTC_LOG, ("compacting LOH due to hard limit"));
15296 #ifdef BGC_SERVO_TUNING
15297 if (bgc_tuning::should_trigger_ngc2())
15299 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_servo_ngc);
15300 n = max_generation;
15301 *blocking_collection_p = TRUE;
15304 if ((n < max_generation) && !gc_heap::background_running_p() &&
15305 bgc_tuning::stepping_trigger (settings.entry_memory_load, get_current_gc_index (max_generation)))
15307 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_servo_initial);
15308 n = max_generation;
15309 saved_bgc_tuning_reason = reason_bgc_stepping;
15312 if ((n < max_generation) && bgc_tuning::should_trigger_bgc())
15314 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_servo_bgc);
15315 n = max_generation;
15318 if (n == (max_generation - 1))
15320 if (bgc_tuning::should_delay_alloc (max_generation))
15322 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_servo_postpone);
15326 #endif //BGC_SERVO_TUNING
15328 if ((n == max_generation) && (*blocking_collection_p == FALSE))
15330 // If we are doing a gen2 we should reset elevation regardless and let the gen2
15331 // decide if we should lock again or in the bgc case by design we will not retract
15333 settings.should_lock_elevation = FALSE;
15334 settings.elevation_locked_count = 0;
15335 dprintf (1, ("doing bgc, reset elevation"));
15339 #ifdef BACKGROUND_GC
15340 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15341 // generations to be collected,
15343 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
15344 // things that need to be fixed in this code block.
15345 if (n_original != max_generation &&
15346 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15348 #ifndef FEATURE_REDHAWK
15349 if (*blocking_collection_p)
15351 // We call StressHeap() a lot for Concurrent GC Stress. However,
15352 // if we can not do a concurrent collection, no need to stress anymore.
15353 // @TODO: Enable stress when the memory pressure goes down again
15354 GCStressPolicy::GlobalDisable();
15357 #endif // !FEATURE_REDHAWK
15359 gc_data_global.gen_to_condemn_reasons.set_condition(gen_joined_stress);
15360 n = max_generation;
15363 #endif //BACKGROUND_GC
15364 #endif //STRESS_HEAP
15370 size_t get_survived_size (gc_history_per_heap* hist)
15372 size_t surv_size = 0;
15373 gc_generation_data* gen_data;
15375 for (int gen_number = 0; gen_number < total_generation_count; gen_number++)
15377 gen_data = &(hist->gen_data[gen_number]);
15378 surv_size += (gen_data->size_after -
15379 gen_data->free_list_space_after -
15380 gen_data->free_obj_space_after);
15386 size_t gc_heap::get_total_survived_size()
15388 size_t total_surv_size = 0;
15389 #ifdef MULTIPLE_HEAPS
15390 for (int i = 0; i < gc_heap::n_heaps; i++)
15392 gc_heap* hp = gc_heap::g_heaps[i];
15393 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
15394 total_surv_size += get_survived_size (current_gc_data_per_heap);
15397 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15398 total_surv_size = get_survived_size (current_gc_data_per_heap);
15399 #endif //MULTIPLE_HEAPS
15400 return total_surv_size;
15403 size_t gc_heap::get_total_allocated_since_last_gc()
15405 size_t total_allocated_size = 0;
15406 #ifdef MULTIPLE_HEAPS
15407 for (int i = 0; i < gc_heap::n_heaps; i++)
15409 gc_heap* hp = gc_heap::g_heaps[i];
15410 total_allocated_size += hp->allocated_since_last_gc;
15411 hp->allocated_since_last_gc = 0;
15414 total_allocated_size = allocated_since_last_gc;
15415 allocated_since_last_gc = 0;
15416 #endif //MULTIPLE_HEAPS
15417 return total_allocated_size;
15420 // Gets what's allocated on both SOH, LOH, etc that hasn't been collected.
15421 size_t gc_heap::get_current_allocated()
15423 dynamic_data* dd = dynamic_data_of (0);
15424 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
15425 for (int i = uoh_start_generation; i < total_generation_count; i++)
15427 dynamic_data* dd = dynamic_data_of (i);
15428 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
15430 return current_alloc;
15433 size_t gc_heap::get_total_allocated()
15435 size_t total_current_allocated = 0;
15436 #ifdef MULTIPLE_HEAPS
15437 for (int i = 0; i < gc_heap::n_heaps; i++)
15439 gc_heap* hp = gc_heap::g_heaps[i];
15440 total_current_allocated += hp->get_current_allocated();
15443 total_current_allocated = get_current_allocated();
15444 #endif //MULTIPLE_HEAPS
15445 return total_current_allocated;
15448 size_t gc_heap::get_total_promoted()
15450 size_t total_promoted_size = 0;
15451 int highest_gen = ((settings.condemned_generation == max_generation) ?
15452 (total_generation_count - 1) : settings.condemned_generation);
15453 #ifdef MULTIPLE_HEAPS
15454 for (int i = 0; i < gc_heap::n_heaps; i++)
15456 gc_heap* hp = gc_heap::g_heaps[i];
15457 #else //MULTIPLE_HEAPS
15459 gc_heap* hp = pGenGCHeap;
15460 #endif //MULTIPLE_HEAPS
15461 for (int gen_number = 0; gen_number <= highest_gen; gen_number++)
15463 total_promoted_size += dd_promoted_size (hp->dynamic_data_of (gen_number));
15466 return total_promoted_size;
15469 #ifdef BGC_SERVO_TUNING
15470 size_t gc_heap::get_total_generation_size (int gen_number)
15472 size_t total_generation_size = 0;
15473 #ifdef MULTIPLE_HEAPS
15474 for (int i = 0; i < gc_heap::n_heaps; i++)
15476 gc_heap* hp = gc_heap::g_heaps[i];
15477 #else //MULTIPLE_HEAPS
15479 gc_heap* hp = pGenGCHeap;
15480 #endif //MULTIPLE_HEAPS
15482 total_generation_size += hp->generation_size (gen_number);
15484 return total_generation_size;
15487 // gets all that's allocated into the gen. This is only used for gen2/3
15488 // for servo tuning.
15489 size_t gc_heap::get_total_servo_alloc (int gen_number)
15491 size_t total_alloc = 0;
15493 #ifdef MULTIPLE_HEAPS
15494 for (int i = 0; i < gc_heap::n_heaps; i++)
15496 gc_heap* hp = gc_heap::g_heaps[i];
15497 #else //MULTIPLE_HEAPS
15499 gc_heap* hp = pGenGCHeap;
15500 #endif //MULTIPLE_HEAPS
15501 generation* gen = hp->generation_of (gen_number);
15502 total_alloc += generation_free_list_allocated (gen);
15503 total_alloc += generation_end_seg_allocated (gen);
15504 total_alloc += generation_condemned_allocated (gen);
15505 total_alloc += generation_sweep_allocated (gen);
15508 return total_alloc;
15511 size_t gc_heap::get_total_bgc_promoted()
15513 size_t total_bgc_promoted = 0;
15514 #ifdef MULTIPLE_HEAPS
15515 int num_heaps = gc_heap::n_heaps;
15516 #else //MULTIPLE_HEAPS
15518 #endif //MULTIPLE_HEAPS
15520 for (int i = 0; i < num_heaps; i++)
15522 total_bgc_promoted += bpromoted_bytes (i);
15524 return total_bgc_promoted;
15527 // This is called after compute_new_dynamic_data is called, at which point
15528 // dd_current_size is calculated.
15529 size_t gc_heap::get_total_surv_size (int gen_number)
15531 size_t total_surv_size = 0;
15532 #ifdef MULTIPLE_HEAPS
15533 for (int i = 0; i < gc_heap::n_heaps; i++)
15535 gc_heap* hp = gc_heap::g_heaps[i];
15536 #else //MULTIPLE_HEAPS
15538 gc_heap* hp = pGenGCHeap;
15539 #endif //MULTIPLE_HEAPS
15540 total_surv_size += dd_current_size (hp->dynamic_data_of (gen_number));
15542 return total_surv_size;
15545 size_t gc_heap::get_total_begin_data_size (int gen_number)
15547 size_t total_begin_data_size = 0;
15548 #ifdef MULTIPLE_HEAPS
15549 for (int i = 0; i < gc_heap::n_heaps; i++)
15551 gc_heap* hp = gc_heap::g_heaps[i];
15552 #else //MULTIPLE_HEAPS
15554 gc_heap* hp = pGenGCHeap;
15555 #endif //MULTIPLE_HEAPS
15557 total_begin_data_size += dd_begin_data_size (hp->dynamic_data_of (gen_number));
15559 return total_begin_data_size;
15562 size_t gc_heap::get_total_generation_fl_size (int gen_number)
15564 size_t total_generation_fl_size = 0;
15565 #ifdef MULTIPLE_HEAPS
15566 for (int i = 0; i < gc_heap::n_heaps; i++)
15568 gc_heap* hp = gc_heap::g_heaps[i];
15569 #else //MULTIPLE_HEAPS
15571 gc_heap* hp = pGenGCHeap;
15572 #endif //MULTIPLE_HEAPS
15573 total_generation_fl_size += generation_free_list_space (hp->generation_of (gen_number));
15575 return total_generation_fl_size;
15578 size_t gc_heap::get_current_gc_index (int gen_number)
15580 #ifdef MULTIPLE_HEAPS
15581 gc_heap* hp = gc_heap::g_heaps[0];
15582 return dd_collection_count (hp->dynamic_data_of (gen_number));
15584 return dd_collection_count (dynamic_data_of (gen_number));
15585 #endif //MULTIPLE_HEAPS
15587 #endif //BGC_SERVO_TUNING
15589 size_t gc_heap::current_generation_size (int gen_number)
15591 dynamic_data* dd = dynamic_data_of (gen_number);
15592 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
15593 - dd_new_allocation (dd));
15599 #pragma warning(push)
15600 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
15604 This is called by when we are actually doing a GC, or when we are just checking whether
15605 we would do a full blocking GC, in which case check_only_p is TRUE.
15607 The difference between calling this with check_only_p TRUE and FALSE is that when it's
15609 settings.reason is ignored
15610 budgets are not checked (since they are checked before this is called)
15611 it doesn't change anything non local like generation_skip_ratio
15613 int gc_heap::generation_to_condemn (int n_initial,
15614 BOOL* blocking_collection_p,
15615 BOOL* elevation_requested_p,
15618 gc_mechanisms temp_settings = settings;
15619 gen_to_condemn_tuning temp_condemn_reasons;
15620 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
15621 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
15624 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
15626 assert (n_initial >= 1);
15629 assert (settings.reason != reason_empty);
15632 local_condemn_reasons->init();
15636 if (heap_number == 0)
15638 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
15642 BOOL low_memory_detected = g_low_memory_status;
15643 uint32_t memory_load = 0;
15644 uint64_t available_physical = 0;
15645 uint64_t available_page_file = 0;
15646 BOOL check_memory = FALSE;
15647 BOOL high_fragmentation = FALSE;
15648 BOOL v_high_memory_load = FALSE;
15649 BOOL high_memory_load = FALSE;
15650 BOOL low_ephemeral_space = FALSE;
15651 BOOL evaluate_elevation = TRUE;
15652 *elevation_requested_p = FALSE;
15653 *blocking_collection_p = FALSE;
15655 BOOL check_max_gen_alloc = TRUE;
15659 #endif //STRESS_HEAP
15663 dd_fragmentation (dynamic_data_of (0)) =
15664 generation_free_list_space (youngest_generation) +
15665 generation_free_obj_space (youngest_generation);
15667 for (int i = uoh_start_generation; i < total_generation_count; i++)
15669 dd_fragmentation (dynamic_data_of (i)) =
15670 generation_free_list_space (generation_of (i)) +
15671 generation_free_obj_space (generation_of (i));
15674 //save new_allocation
15675 for (i = 0; i < total_generation_count; i++)
15677 dynamic_data* dd = dynamic_data_of (i);
15678 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
15680 dd_new_allocation (dd),
15681 dd_desired_allocation (dd)));
15682 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
15685 local_condemn_reasons->set_gen (gen_initial, n);
15688 #ifdef BACKGROUND_GC
15689 if (gc_heap::background_running_p()
15690 #ifdef BGC_SERVO_TUNING
15691 || bgc_tuning::fl_tuning_triggered
15692 || (bgc_tuning::enable_fl_tuning && bgc_tuning::use_stepping_trigger_p)
15693 #endif //BGC_SERVO_TUNING
15696 check_max_gen_alloc = FALSE;
15698 #endif //BACKGROUND_GC
15700 if (check_max_gen_alloc)
15702 //figure out if UOH objects need to be collected.
15703 for (int i = uoh_start_generation; i < total_generation_count; i++)
15705 if (get_new_allocation (i) <= 0)
15707 n = max_generation;
15708 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15709 dprintf (BGC_TUNING_LOG, ("BTL[GTC]: trigger based on gen%d b: %Id",
15711 get_new_allocation (i)));
15716 //figure out which generation ran out of allocation
15717 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
15719 if (get_new_allocation (i) <= 0)
15722 if (n == max_generation)
15724 dprintf (BGC_TUNING_LOG, ("BTL[GTC]: trigger based on gen2 b: %Id",
15725 get_new_allocation (max_generation)));
15735 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15738 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (loh_generation) <= 0) ? 3 : n)));
15742 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
15743 //time based tuning
15744 // if enough time has elapsed since the last gc
15745 // and the number of gc is too low (1/10 of lower gen) then collect
15746 // This should also be enabled if we have memory concerns
15747 int n_time_max = max_generation;
15751 if (!check_max_gen_alloc)
15753 n_time_max = max_generation - 1;
15757 if ((local_settings->pause_mode == pause_interactive) ||
15758 (local_settings->pause_mode == pause_sustained_low_latency))
15760 dynamic_data* dd0 = dynamic_data_of (0);
15761 uint64_t now = GetHighPrecisionTimeStamp();
15763 for (i = (temp_gen+1); i <= n_time_max; i++)
15765 dynamic_data* dd = dynamic_data_of (i);
15766 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
15767 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
15768 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
15770 n = min (i, n_time_max);
15771 dprintf (GTC_LOG, ("time %d", n));
15776 local_condemn_reasons->set_gen (gen_time_tuning, n);
15777 if (n == max_generation)
15779 dprintf (BGC_TUNING_LOG, ("BTL[GTC]: trigger based on time"));
15786 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
15788 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
15790 if (n < (max_generation - 1))
15792 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
15794 n = max (n, max_generation - 1);
15795 local_settings->promotion = TRUE;
15796 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
15797 heap_number, generation_skip_ratio, n));
15798 local_condemn_reasons->set_condition (gen_low_card_p);
15804 generation_skip_ratio = 100;
15807 if (dt_low_ephemeral_space_p (check_only_p ?
15808 tuning_deciding_full_gc :
15809 tuning_deciding_condemned_gen))
15811 low_ephemeral_space = TRUE;
15813 n = max (n, max_generation - 1);
15814 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
15815 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
15817 if (!provisional_mode_triggered)
15819 #ifdef BACKGROUND_GC
15820 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
15821 #endif //BACKGROUND_GC
15823 //It is better to defragment first if we are running out of space for
15824 //the ephemeral generation but we have enough fragmentation to make up for it
15825 //in the non ephemeral generation. Essentially we are trading a gen2 for
15826 // having to expand heap in ephemeral collections.
15827 if (dt_high_frag_p (tuning_deciding_condemned_gen,
15828 max_generation - 1,
15831 high_fragmentation = TRUE;
15832 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
15833 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
15839 //figure out which ephemeral generation is too fragmented
15841 for (i = n+1; i < max_generation; i++)
15843 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
15845 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
15852 if (low_ephemeral_space)
15855 local_settings->promotion = TRUE;
15860 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
15865 if (settings.pause_mode == pause_low_latency)
15867 if (!is_induced (settings.reason))
15869 n = min (n, max_generation - 1);
15870 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
15871 evaluate_elevation = FALSE;
15877 // It's hard to catch when we get to the point that the memory load is so high
15878 // we get an induced GC from the finalizer thread so we are checking the memory load
15879 // for every gen0 GC.
15880 check_memory = (check_only_p ?
15882 ((n >= 1) || low_memory_detected));
15886 //find out if we are short on memory
15887 get_memory_info (&memory_load, &available_physical, &available_page_file);
15888 if (heap_number == 0)
15890 dprintf (GTC_LOG, ("ml: %d", memory_load));
15893 // Need to get it early enough for all heaps to use.
15894 local_settings->entry_available_physical_mem = available_physical;
15895 local_settings->entry_memory_load = memory_load;
15897 // @TODO: Force compaction more often under GCSTRESS
15898 if (memory_load >= high_memory_load_th || low_memory_detected)
15900 #ifdef SIMPLE_DPRINTF
15901 // stress log can't handle any parameter that's bigger than a void*.
15902 if (heap_number == 0)
15904 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
15906 #endif //SIMPLE_DPRINTF
15908 high_memory_load = TRUE;
15910 if (memory_load >= v_high_memory_load_th || low_memory_detected)
15912 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
15913 // gen1/gen0 may take a lot more memory than gen2.
15914 if (!high_fragmentation)
15916 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
15918 v_high_memory_load = TRUE;
15922 if (!high_fragmentation)
15924 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
15928 if (high_fragmentation)
15930 if (high_memory_load)
15932 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
15934 else if (v_high_memory_load)
15936 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
15942 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
15943 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
15944 high_fragmentation));
15946 if (should_expand_in_full_gc)
15948 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
15949 *blocking_collection_p = TRUE;
15950 evaluate_elevation = FALSE;
15951 n = max_generation;
15952 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
15955 if (last_gc_before_oom)
15957 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
15958 n = max_generation;
15959 *blocking_collection_p = TRUE;
15961 if ((local_settings->reason == reason_oos_loh) ||
15962 (local_settings->reason == reason_alloc_loh))
15964 evaluate_elevation = FALSE;
15967 local_condemn_reasons->set_condition (gen_before_oom);
15972 if (is_induced_blocking (settings.reason) &&
15973 n_initial == max_generation
15974 IN_STRESS_HEAP( && !settings.stress_induced ))
15976 if (heap_number == 0)
15978 dprintf (GTC_LOG, ("induced - BLOCK"));
15981 *blocking_collection_p = TRUE;
15982 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
15983 evaluate_elevation = FALSE;
15986 if (settings.reason == reason_induced_noforce)
15988 local_condemn_reasons->set_condition (gen_induced_noforce_p);
15989 evaluate_elevation = FALSE;
15993 if (!provisional_mode_triggered && evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
15995 *elevation_requested_p = TRUE;
15997 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
15998 if (high_memory_load || v_high_memory_load)
16000 dynamic_data* dd_max = dynamic_data_of (max_generation);
16001 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
16003 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
16004 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
16005 n = max_generation;
16006 local_condemn_reasons->set_condition (gen_almost_max_alloc);
16010 if (n <= max_generation)
16012 #endif // HOST_64BIT
16013 if (high_fragmentation)
16015 //elevate to max_generation
16016 n = max_generation;
16017 dprintf (GTC_LOG, ("h%d: f full", heap_number));
16019 #ifdef BACKGROUND_GC
16020 if (high_memory_load || v_high_memory_load)
16022 // For background GC we want to do blocking collections more eagerly because we don't
16023 // want to get into the situation where the memory load becomes high while we are in
16024 // a background GC and we'd have to wait for the background GC to finish to start
16025 // a blocking collection (right now the implemenation doesn't handle converting
16026 // a background GC to a blocking collection midway.
16027 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
16028 *blocking_collection_p = TRUE;
16031 if (v_high_memory_load)
16033 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
16034 *blocking_collection_p = TRUE;
16036 #endif //BACKGROUND_GC
16040 n = max (n, max_generation - 1);
16041 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
16045 #endif // HOST_64BIT
16048 if (!provisional_mode_triggered && (n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
16050 #ifdef BGC_SERVO_TUNING
16051 if (!bgc_tuning::enable_fl_tuning)
16052 #endif //BGC_SERVO_TUNING
16054 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
16055 heap_number, n_alloc));
16056 if (get_new_allocation (max_generation) <= 0)
16058 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
16059 n = max_generation;
16060 local_condemn_reasons->set_condition (gen_max_gen1);
16065 //figure out if max_generation is too fragmented -> blocking collection
16066 if (!provisional_mode_triggered
16067 #ifdef BGC_SERVO_TUNING
16068 && !bgc_tuning::enable_fl_tuning
16069 #endif //BGC_SERVO_TUNING
16070 && (n == max_generation))
16072 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
16074 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
16075 local_condemn_reasons->set_condition (gen_max_high_frag_p);
16076 if (local_settings->pause_mode != pause_sustained_low_latency)
16078 *blocking_collection_p = TRUE;
16083 #ifdef BACKGROUND_GC
16084 if ((n == max_generation) && !(*blocking_collection_p))
16086 if (heap_number == 0)
16088 BOOL bgc_heap_too_small = TRUE;
16089 size_t gen2size = 0;
16090 size_t gen3size = 0;
16091 #ifdef MULTIPLE_HEAPS
16092 for (int i = 0; i < n_heaps; i++)
16094 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
16095 ((g_heaps[i]->current_generation_size (loh_generation)) > bgc_min_per_heap) ||
16096 ((g_heaps[i]->current_generation_size (poh_generation)) > bgc_min_per_heap))
16098 bgc_heap_too_small = FALSE;
16102 #else //MULTIPLE_HEAPS
16103 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
16104 (current_generation_size (loh_generation) > bgc_min_per_heap) ||
16105 (current_generation_size (poh_generation) > bgc_min_per_heap))
16107 bgc_heap_too_small = FALSE;
16109 #endif //MULTIPLE_HEAPS
16111 if (bgc_heap_too_small)
16113 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
16116 // do not turn stress-induced collections into blocking GCs
16117 if (!settings.stress_induced)
16118 #endif //STRESS_HEAP
16120 *blocking_collection_p = TRUE;
16123 local_condemn_reasons->set_condition (gen_gen2_too_small);
16127 #endif //BACKGROUND_GC
16133 #ifdef BACKGROUND_GC
16134 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
16135 // generations to be collected,
16137 if (orig_gen != max_generation &&
16138 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
16140 *elevation_requested_p = FALSE;
16142 #endif //BACKGROUND_GC
16143 #endif //STRESS_HEAP
16147 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
16150 local_condemn_reasons->set_gen (gen_final_per_heap, n);
16151 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
16154 local_condemn_reasons->print (heap_number);
16157 if ((local_settings->reason == reason_oos_soh) ||
16158 (local_settings->reason == reason_oos_loh))
16168 #pragma warning(pop)
16172 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
16174 // if the memory load is higher, the threshold we'd want to collect gets lower.
16175 size_t min_mem_based_on_available =
16176 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
16178 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
16179 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
16181 #ifdef SIMPLE_DPRINTF
16182 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
16183 min_mem_based_on_available, ten_percent_size, three_percent_mem));
16184 #endif //SIMPLE_DPRINTF
16185 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
16189 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
16191 return min (available_mem, (256*1024*1024)) / num_heaps;
16195 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
16199 #ifdef BACKGROUND_GC
16200 void gc_heap::init_background_gc ()
16202 //reset the allocation so foreground gc can allocate into older (max_generation) generation
16203 generation* gen = generation_of (max_generation);
16204 generation_allocation_pointer (gen)= 0;
16205 generation_allocation_limit (gen) = 0;
16206 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
16208 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
16210 //reset the plan allocation for each segment
16211 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
16212 seg = heap_segment_next_rw (seg))
16214 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
16217 if (heap_number == 0)
16219 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
16221 background_saved_lowest_address,
16222 background_saved_highest_address));
16226 #endif //BACKGROUND_GC
16229 void fire_drain_mark_list_event (size_t mark_list_objects)
16231 FIRE_EVENT(BGCDrainMark, mark_list_objects);
16235 void fire_revisit_event (size_t dirtied_pages,
16236 size_t marked_objects,
16237 BOOL large_objects_p)
16239 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
16243 void fire_overflow_event (uint8_t* overflow_min,
16244 uint8_t* overflow_max,
16245 size_t marked_objects,
16248 FIRE_EVENT(BGCOverflow_V1, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, gen_number == loh_generation, gen_number);
16251 void gc_heap::concurrent_print_time_delta (const char* msg)
16254 uint64_t current_time = GetHighPrecisionTimeStamp();
16255 size_t elapsed_time_ms = (size_t)((current_time - time_bgc_last) / 1000);
16256 time_bgc_last = current_time;
16258 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time_ms));
16260 UNREFERENCED_PARAMETER(msg);
16264 void gc_heap::free_list_info (int gen_num, const char* msg)
16266 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
16267 dprintf (3, ("h%d: %s", heap_number, msg));
16268 for (int i = 0; i < total_generation_count; i++)
16270 generation* gen = generation_of (i);
16271 if ((generation_allocation_size (gen) == 0) &&
16272 (generation_free_list_space (gen) == 0) &&
16273 (generation_free_obj_space (gen) == 0))
16275 // don't print if everything is 0.
16279 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
16281 generation_allocation_size (gen),
16282 generation_free_list_space (gen),
16283 generation_free_obj_space (gen)));
16287 UNREFERENCED_PARAMETER(gen_num);
16288 UNREFERENCED_PARAMETER(msg);
16289 #endif // BACKGROUND_GC && TRACE_GC
16292 void gc_heap::update_collection_counts_for_no_gc()
16294 assert (settings.pause_mode == pause_no_gc);
16296 settings.condemned_generation = max_generation;
16297 #ifdef MULTIPLE_HEAPS
16298 for (int i = 0; i < n_heaps; i++)
16299 g_heaps[i]->update_collection_counts();
16300 #else //MULTIPLE_HEAPS
16301 update_collection_counts();
16302 #endif //MULTIPLE_HEAPS
16304 full_gc_counts[gc_type_blocking]++;
16307 BOOL gc_heap::should_proceed_with_gc()
16309 if (gc_heap::settings.pause_mode == pause_no_gc)
16311 if (current_no_gc_region_info.started)
16313 // The no_gc mode was already in progress yet we triggered another GC,
16314 // this effectively exits the no_gc mode.
16315 restore_data_for_no_gc();
16318 return should_proceed_for_no_gc();
16324 void gc_heap::update_end_gc_time_per_heap()
16326 for (int gen_number = 0; gen_number <= settings.condemned_generation; gen_number++)
16328 dynamic_data* dd = dynamic_data_of (gen_number);
16329 dd_gc_elapsed_time (dd) = (size_t)(end_gc_time - dd_time_clock (dd));
16333 void gc_heap::update_end_ngc_time()
16335 end_gc_time = GetHighPrecisionTimeStamp();
16336 #ifdef HEAP_BALANCE_INSTRUMENTATION
16337 last_gc_end_time_us = end_gc_time;
16338 dprintf (HEAP_BALANCE_LOG, ("[GC#%Id-%Id-%Id]", settings.gc_index,
16339 (last_gc_end_time_us - dd_time_clock (dynamic_data_of (0))),
16340 dd_time_clock (dynamic_data_of (0))));
16341 #endif //HEAP_BALANCE_INSTRUMENTATION
16344 //internal part of gc used by the serial and concurrent version
16345 void gc_heap::gc1()
16347 #ifdef BACKGROUND_GC
16348 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
16349 #endif //BACKGROUND_GC
16351 verify_soh_segment_list();
16353 int n = settings.condemned_generation;
16355 if (settings.reason == reason_pm_full_gc)
16357 assert (n == max_generation);
16360 gen_to_condemn_tuning* local_condemn_reasons = &(get_gc_data_per_heap()->gen_to_condemn_reasons);
16361 local_condemn_reasons->init();
16362 local_condemn_reasons->set_gen (gen_initial, n);
16363 local_condemn_reasons->set_gen (gen_final_per_heap, n);
16366 update_collection_counts ();
16368 #ifdef BACKGROUND_GC
16369 bgc_alloc_lock->check();
16370 #endif //BACKGROUND_GC
16372 free_list_info (max_generation, "beginning");
16374 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
16376 assert (g_gc_card_table == card_table);
16378 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
16379 assert (g_gc_card_bundle_table == card_bundle_table);
16383 if (n == max_generation)
16385 gc_low = lowest_address;
16386 gc_high = highest_address;
16390 gc_low = generation_allocation_start (generation_of (n));
16391 gc_high = heap_segment_reserved (ephemeral_heap_segment);
16393 #ifdef BACKGROUND_GC
16394 if (settings.concurrent)
16397 time_bgc_last = GetHighPrecisionTimeStamp();
16400 FIRE_EVENT(BGCBegin);
16402 concurrent_print_time_delta ("BGC");
16404 concurrent_print_time_delta ("RW");
16405 background_mark_phase();
16406 free_list_info (max_generation, "after mark phase");
16408 background_sweep();
16409 free_list_info (max_generation, "after sweep phase");
16412 #endif //BACKGROUND_GC
16414 mark_phase (n, FALSE);
16416 GCScan::GcRuntimeStructuresValid (FALSE);
16418 GCScan::GcRuntimeStructuresValid (TRUE);
16422 //adjust the allocation size from the pinned quantities.
16423 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
16425 generation* gn = generation_of (gen_number);
16426 if (settings.compaction)
16428 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
16429 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
16433 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
16434 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
16436 generation_pinned_allocation_sweep_size (gn) = 0;
16437 generation_pinned_allocation_compact_size (gn) = 0;
16440 #ifdef BACKGROUND_GC
16441 if (settings.concurrent)
16443 dynamic_data* dd = dynamic_data_of (n);
16444 end_gc_time = GetHighPrecisionTimeStamp();
16445 dd_gc_elapsed_time (dd) = (size_t)(end_gc_time - dd_time_clock (dd));
16447 #ifdef HEAP_BALANCE_INSTRUMENTATION
16448 if (heap_number == 0)
16450 last_gc_end_time_us = end_gc_time;
16451 dprintf (HEAP_BALANCE_LOG, ("[GC#%Id-%Id-BGC]", settings.gc_index, dd_gc_elapsed_time (dd)));
16453 #endif //HEAP_BALANCE_INSTRUMENTATION
16455 free_list_info (max_generation, "after computing new dynamic data");
16457 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
16459 for (int gen_number = 0; gen_number < max_generation; gen_number++)
16461 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
16462 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
16463 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
16464 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
16465 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
16469 #endif //BACKGROUND_GC
16471 free_list_info (max_generation, "end");
16472 for (int gen_number = 0; gen_number <= n; gen_number++)
16474 compute_new_dynamic_data (gen_number);
16477 if (n != max_generation)
16479 for (int gen_number = (n + 1); gen_number < total_generation_count; gen_number++)
16481 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
16482 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
16483 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
16487 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
16489 free_list_info (max_generation, "after computing new dynamic data");
16492 if (n < max_generation)
16494 compute_promoted_allocation (1 + n);
16496 dynamic_data* dd = dynamic_data_of (1 + n);
16497 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
16498 generation_free_obj_space (generation_of (1 + n));
16500 #ifdef BACKGROUND_GC
16501 if (current_c_gc_state != c_gc_state_planning)
16502 #endif //BACKGROUND_GC
16504 if (settings.promotion)
16506 dd_fragmentation (dd) = new_fragmentation;
16510 //assert (dd_fragmentation (dd) == new_fragmentation);
16515 #ifdef BACKGROUND_GC
16516 if (!settings.concurrent)
16517 #endif //BACKGROUND_GC
16519 #ifndef FEATURE_REDHAWK
16520 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
16521 assert(GCToEEInterface::IsGCThread());
16522 #endif // FEATURE_REDHAWK
16523 adjust_ephemeral_limits();
16526 #ifdef BACKGROUND_GC
16527 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
16528 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
16529 #endif //BACKGROUND_GC
16531 if (fgn_maxgen_percent)
16533 if (settings.condemned_generation == (max_generation - 1))
16535 check_for_full_gc (max_generation - 1, 0);
16537 else if (settings.condemned_generation == max_generation)
16539 if (full_gc_approach_event_set
16540 #ifdef MULTIPLE_HEAPS
16541 && (heap_number == 0)
16542 #endif //MULTIPLE_HEAPS
16545 dprintf (2, ("FGN-GC: setting gen2 end event"));
16547 full_gc_approach_event.Reset();
16548 #ifdef BACKGROUND_GC
16549 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
16550 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
16551 #endif //BACKGROUND_GC
16552 full_gc_end_event.Set();
16553 full_gc_approach_event_set = false;
16558 #ifdef BACKGROUND_GC
16559 if (!settings.concurrent)
16560 #endif //BACKGROUND_GC
16562 //decide on the next allocation quantum
16563 if (alloc_contexts_used >= 1)
16565 allocation_quantum = Align (min ((size_t)CLR_SIZE,
16566 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
16567 get_alignment_constant(FALSE));
16568 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
16572 descr_generations (FALSE);
16574 verify_soh_segment_list();
16576 #ifdef BACKGROUND_GC
16577 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
16578 #endif //BACKGROUND_GC
16580 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
16583 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
16584 // value. If we ever allow randomly adjusting this as the process runs,
16585 // we cannot call it this way as joins need to match - we must have the same
16586 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
16587 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
16589 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
16590 || (bgc_heap_walk_for_etw_p && settings.concurrent)
16594 #ifdef BACKGROUND_GC
16595 bool cooperative_mode = true;
16597 if (settings.concurrent)
16599 cooperative_mode = enable_preemptive ();
16601 #ifdef MULTIPLE_HEAPS
16602 bgc_t_join.join(this, gc_join_suspend_ee_verify);
16603 if (bgc_t_join.joined())
16605 bgc_threads_sync_event.Reset();
16607 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
16608 bgc_t_join.restart();
16610 if (heap_number == 0)
16613 bgc_threads_sync_event.Set();
16617 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16618 dprintf (2, ("bgc_threads_sync_event is signalled"));
16620 #else //MULTIPLE_HEAPS
16622 #endif //MULTIPLE_HEAPS
16624 //fix the allocation area so verify_heap can proceed.
16625 fix_allocation_contexts (FALSE);
16627 #endif //BACKGROUND_GC
16629 #ifdef BACKGROUND_GC
16630 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
16631 #ifdef FEATURE_EVENT_TRACE
16632 if (bgc_heap_walk_for_etw_p && settings.concurrent)
16634 GCToEEInterface::DiagWalkBGCSurvivors(__this);
16636 #ifdef MULTIPLE_HEAPS
16637 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
16638 if (bgc_t_join.joined())
16640 bgc_t_join.restart();
16642 #endif // MULTIPLE_HEAPS
16644 #endif // FEATURE_EVENT_TRACE
16645 #endif //BACKGROUND_GC
16648 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
16649 verify_heap (FALSE);
16650 #endif // VERIFY_HEAP
16652 #ifdef BACKGROUND_GC
16653 if (settings.concurrent)
16655 repair_allocation_contexts (TRUE);
16657 #ifdef MULTIPLE_HEAPS
16658 bgc_t_join.join(this, gc_join_restart_ee_verify);
16659 if (bgc_t_join.joined())
16661 bgc_threads_sync_event.Reset();
16663 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
16664 bgc_t_join.restart();
16666 if (heap_number == 0)
16669 bgc_threads_sync_event.Set();
16673 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16674 dprintf (2, ("bgc_threads_sync_event is signalled"));
16676 #else //MULTIPLE_HEAPS
16678 #endif //MULTIPLE_HEAPS
16680 disable_preemptive (cooperative_mode);
16682 #endif //BACKGROUND_GC
16684 #endif //VERIFY_HEAP || (FEATURE_EVENT_TRACE && BACKGROUND_GC)
16686 #ifdef MULTIPLE_HEAPS
16687 if (!settings.concurrent)
16689 gc_t_join.join(this, gc_join_done);
16690 if (gc_t_join.joined ())
16692 gc_heap::internal_gc_done = false;
16694 //equalize the new desired size of the generations
16695 int limit = settings.condemned_generation;
16696 if (limit == max_generation)
16698 limit = total_generation_count-1;
16700 for (int gen = 0; gen <= limit; gen++)
16702 size_t total_desired = 0;
16704 for (int i = 0; i < gc_heap::n_heaps; i++)
16706 gc_heap* hp = gc_heap::g_heaps[i];
16707 dynamic_data* dd = hp->dynamic_data_of (gen);
16708 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
16709 if (temp_total_desired < total_desired)
16712 total_desired = (size_t)MAX_PTR;
16715 total_desired = temp_total_desired;
16718 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
16719 get_alignment_constant (gen <= max_generation));
16723 #if 1 //subsumed by the linear allocation model
16724 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16725 // apply some smoothing.
16726 size_t smoothing = 3; // exponential smoothing factor
16727 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
16728 dprintf (HEAP_BALANCE_LOG, ("TEMPsn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
16729 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
16732 if (!heap_hard_limit)
16734 // if desired_per_heap is close to min_gc_size, trim it
16735 // down to min_gc_size to stay in the cache
16736 gc_heap* hp = gc_heap::g_heaps[0];
16737 dynamic_data* dd = hp->dynamic_data_of (gen);
16738 size_t min_gc_size = dd_min_size(dd);
16739 // if min GC size larger than true on die cache, then don't bother
16740 // limiting the desired size
16741 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
16742 desired_per_heap <= 2*min_gc_size)
16744 desired_per_heap = min_gc_size;
16748 desired_per_heap = joined_youngest_desired (desired_per_heap);
16749 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
16750 #endif // HOST_64BIT
16751 gc_data_global.final_youngest_desired = desired_per_heap;
16753 #if 1 //subsumed by the linear allocation model
16754 if (gen >= uoh_start_generation)
16756 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16757 // apply some smoothing.
16758 static size_t smoothed_desired_per_heap_uoh = 0;
16759 size_t smoothing = 3; // exponential smoothing factor
16760 size_t uoh_count = dd_collection_count (dynamic_data_of (max_generation));
16761 if (smoothing > uoh_count)
16762 smoothing = uoh_count;
16763 smoothed_desired_per_heap_uoh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_uoh / smoothing) * (smoothing-1));
16764 dprintf (2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_uoh, desired_per_heap));
16765 desired_per_heap = Align(smoothed_desired_per_heap_uoh, get_alignment_constant (false));
16768 for (int i = 0; i < gc_heap::n_heaps; i++)
16770 gc_heap* hp = gc_heap::g_heaps[i];
16771 dynamic_data* dd = hp->dynamic_data_of (gen);
16772 dd_desired_allocation (dd) = desired_per_heap;
16773 dd_gc_new_allocation (dd) = desired_per_heap;
16774 dd_new_allocation (dd) = desired_per_heap;
16778 hp->fgn_last_alloc = desired_per_heap;
16783 #ifdef FEATURE_LOH_COMPACTION
16784 BOOL all_heaps_compacted_p = TRUE;
16785 #endif //FEATURE_LOH_COMPACTION
16786 int max_gen0_must_clear_bricks = 0;
16787 for (int i = 0; i < gc_heap::n_heaps; i++)
16789 gc_heap* hp = gc_heap::g_heaps[i];
16790 hp->decommit_ephemeral_segment_pages();
16791 hp->rearrange_uoh_segments();
16792 #ifdef FEATURE_LOH_COMPACTION
16793 all_heaps_compacted_p &= hp->loh_compacted_p;
16794 #endif //FEATURE_LOH_COMPACTION
16795 // compute max of gen0_must_clear_bricks over all heaps
16796 max_gen0_must_clear_bricks = max(max_gen0_must_clear_bricks, hp->gen0_must_clear_bricks);
16799 #ifdef FEATURE_LOH_COMPACTION
16800 check_loh_compact_mode (all_heaps_compacted_p);
16801 #endif //FEATURE_LOH_COMPACTION
16803 // if max_gen0_must_clear_bricks > 0, distribute to all heaps -
16804 // if one heap encountered an interior pointer during this GC,
16805 // the next GC might see one on another heap
16806 if (max_gen0_must_clear_bricks > 0)
16808 for (int i = 0; i < gc_heap::n_heaps; i++)
16810 gc_heap* hp = gc_heap::g_heaps[i];
16811 hp->gen0_must_clear_bricks = max_gen0_must_clear_bricks;
16816 update_end_ngc_time();
16817 pm_full_gc_init_or_clear();
16819 gc_t_join.restart();
16822 update_end_gc_time_per_heap();
16823 add_to_history_per_heap();
16824 alloc_context_count = 0;
16825 heap_select::mark_heap (heap_number);
16827 #else //MULTIPLE_HEAPS
16828 gc_data_global.final_youngest_desired =
16829 dd_desired_allocation (dynamic_data_of (0));
16831 check_loh_compact_mode (loh_compacted_p);
16833 decommit_ephemeral_segment_pages();
16836 if (!(settings.concurrent))
16838 rearrange_uoh_segments();
16839 update_end_ngc_time();
16840 update_end_gc_time_per_heap();
16841 add_to_history_per_heap();
16845 pm_full_gc_init_or_clear();
16847 #ifdef BACKGROUND_GC
16848 recover_bgc_settings();
16849 #endif //BACKGROUND_GC
16850 #endif //MULTIPLE_HEAPS
16853 void gc_heap::save_data_for_no_gc()
16855 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
16856 #ifdef MULTIPLE_HEAPS
16857 // This is to affect heap balancing.
16858 for (int i = 0; i < n_heaps; i++)
16860 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
16861 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
16862 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (loh_generation));
16863 dd_min_size (g_heaps[i]->dynamic_data_of (loh_generation)) = 0;
16865 #endif //MULTIPLE_HEAPS
16868 void gc_heap::restore_data_for_no_gc()
16870 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
16871 #ifdef MULTIPLE_HEAPS
16872 for (int i = 0; i < n_heaps; i++)
16874 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
16875 dd_min_size (g_heaps[i]->dynamic_data_of (loh_generation)) = current_no_gc_region_info.saved_gen3_min_size;
16877 #endif //MULTIPLE_HEAPS
16880 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
16881 BOOL loh_size_known,
16883 BOOL disallow_full_blocking)
16885 if (current_no_gc_region_info.started)
16887 return start_no_gc_in_progress;
16890 start_no_gc_region_status status = start_no_gc_success;
16892 save_data_for_no_gc();
16893 settings.pause_mode = pause_no_gc;
16894 current_no_gc_region_info.start_status = start_no_gc_success;
16896 uint64_t allocation_no_gc_loh = 0;
16897 uint64_t allocation_no_gc_soh = 0;
16898 assert(total_size != 0);
16899 if (loh_size_known)
16901 assert(loh_size != 0);
16902 assert(loh_size <= total_size);
16903 allocation_no_gc_loh = loh_size;
16904 allocation_no_gc_soh = total_size - loh_size;
16908 allocation_no_gc_soh = total_size;
16909 allocation_no_gc_loh = total_size;
16912 int soh_align_const = get_alignment_constant (TRUE);
16913 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
16914 size_t size_per_heap = 0;
16915 const double scale_factor = 1.05;
16918 #ifdef MULTIPLE_HEAPS
16919 num_heaps = n_heaps;
16920 #endif // MULTIPLE_HEAPS
16922 uint64_t total_allowed_soh_allocation = (uint64_t)max_soh_allocated * num_heaps;
16924 // In theory, the upper limit here is the physical memory of the machine, not
16925 // SIZE_T_MAX. This is not true today because total_physical_mem can be
16926 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
16927 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
16928 // more freely between branches, it would be good to clean this up to use
16929 // total_physical_mem instead of SIZE_T_MAX.
16930 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
16931 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
16932 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
16933 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
16935 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
16936 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
16938 status = start_no_gc_too_large;
16942 if (allocation_no_gc_soh > 0)
16944 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
16945 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
16948 if (allocation_no_gc_loh > 0)
16950 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
16951 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
16954 if (disallow_full_blocking)
16955 current_no_gc_region_info.minimal_gc_p = TRUE;
16957 if (allocation_no_gc_soh != 0)
16959 current_no_gc_region_info.soh_allocation_size = (size_t)allocation_no_gc_soh;
16960 size_per_heap = current_no_gc_region_info.soh_allocation_size;
16961 #ifdef MULTIPLE_HEAPS
16962 size_per_heap /= n_heaps;
16963 for (int i = 0; i < n_heaps; i++)
16965 // due to heap balancing we need to allow some room before we even look to balance to another heap.
16966 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
16968 #else //MULTIPLE_HEAPS
16969 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
16970 #endif //MULTIPLE_HEAPS
16973 if (allocation_no_gc_loh != 0)
16975 current_no_gc_region_info.loh_allocation_size = (size_t)allocation_no_gc_loh;
16976 size_per_heap = current_no_gc_region_info.loh_allocation_size;
16977 #ifdef MULTIPLE_HEAPS
16978 size_per_heap /= n_heaps;
16979 for (int i = 0; i < n_heaps; i++)
16980 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16981 #else //MULTIPLE_HEAPS
16982 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16983 #endif //MULTIPLE_HEAPS
16987 if (status != start_no_gc_success)
16988 restore_data_for_no_gc();
16992 void gc_heap::handle_failure_for_no_gc()
16994 gc_heap::restore_data_for_no_gc();
16995 // sets current_no_gc_region_info.started to FALSE here.
16996 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16999 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
17001 return current_no_gc_region_info.start_status;
17004 void gc_heap::record_gcs_during_no_gc()
17006 if (current_no_gc_region_info.started)
17008 current_no_gc_region_info.num_gcs++;
17009 if (is_induced (settings.reason))
17010 current_no_gc_region_info.num_gcs_induced++;
17014 BOOL gc_heap::find_loh_free_for_no_gc()
17016 allocator* loh_allocator = generation_allocator (generation_of (loh_generation));
17017 size_t size = loh_allocation_no_gc;
17018 for (unsigned int a_l_idx = loh_allocator->first_suitable_bucket(size); a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
17020 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
17023 size_t free_list_size = unused_array_size(free_list);
17025 if (free_list_size > size)
17027 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
17031 free_list = free_list_slot (free_list);
17038 BOOL gc_heap::find_loh_space_for_no_gc()
17040 saved_loh_segment_no_gc = 0;
17042 if (find_loh_free_for_no_gc())
17045 heap_segment* seg = generation_allocation_segment (generation_of (loh_generation));
17049 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
17050 if (remaining >= loh_allocation_no_gc)
17052 saved_loh_segment_no_gc = seg;
17055 seg = heap_segment_next (seg);
17058 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
17060 // If no full GC is allowed, we try to get a new seg right away.
17061 saved_loh_segment_no_gc = get_segment_for_uoh (loh_generation, get_uoh_seg_size (loh_allocation_no_gc)
17062 #ifdef MULTIPLE_HEAPS
17064 #endif //MULTIPLE_HEAPS
17068 return (saved_loh_segment_no_gc != 0);
17071 BOOL gc_heap::loh_allocated_for_no_gc()
17073 if (!saved_loh_segment_no_gc)
17076 heap_segment* seg = generation_allocation_segment (generation_of (loh_generation));
17079 if (seg == saved_loh_segment_no_gc)
17083 seg = heap_segment_next (seg);
17089 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
17091 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
17092 assert (end_committed <= heap_segment_reserved (seg));
17093 return (grow_heap_segment (seg, end_committed));
17096 void gc_heap::thread_no_gc_loh_segments()
17098 #ifdef MULTIPLE_HEAPS
17099 for (int i = 0; i < n_heaps; i++)
17101 gc_heap* hp = g_heaps[i];
17102 if (hp->loh_allocated_for_no_gc())
17104 hp->thread_uoh_segment (loh_generation, hp->saved_loh_segment_no_gc);
17105 hp->saved_loh_segment_no_gc = 0;
17108 #else //MULTIPLE_HEAPS
17109 if (loh_allocated_for_no_gc())
17111 thread_uoh_segment (loh_generation, saved_loh_segment_no_gc);
17112 saved_loh_segment_no_gc = 0;
17114 #endif //MULTIPLE_HEAPS
17117 void gc_heap::set_loh_allocations_for_no_gc()
17119 if (current_no_gc_region_info.loh_allocation_size != 0)
17121 dynamic_data* dd = dynamic_data_of (loh_generation);
17122 dd_new_allocation (dd) = loh_allocation_no_gc;
17123 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
17127 void gc_heap::set_soh_allocations_for_no_gc()
17129 if (current_no_gc_region_info.soh_allocation_size != 0)
17131 dynamic_data* dd = dynamic_data_of (0);
17132 dd_new_allocation (dd) = soh_allocation_no_gc;
17133 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
17134 #ifdef MULTIPLE_HEAPS
17135 alloc_context_count = 0;
17136 #endif //MULTIPLE_HEAPS
17140 void gc_heap::set_allocations_for_no_gc()
17142 #ifdef MULTIPLE_HEAPS
17143 for (int i = 0; i < n_heaps; i++)
17145 gc_heap* hp = g_heaps[i];
17146 hp->set_loh_allocations_for_no_gc();
17147 hp->set_soh_allocations_for_no_gc();
17149 #else //MULTIPLE_HEAPS
17150 set_loh_allocations_for_no_gc();
17151 set_soh_allocations_for_no_gc();
17152 #endif //MULTIPLE_HEAPS
17155 BOOL gc_heap::should_proceed_for_no_gc()
17157 BOOL gc_requested = FALSE;
17158 BOOL loh_full_gc_requested = FALSE;
17159 BOOL soh_full_gc_requested = FALSE;
17160 BOOL no_gc_requested = FALSE;
17161 BOOL get_new_loh_segments = FALSE;
17163 if (current_no_gc_region_info.soh_allocation_size)
17165 #ifdef MULTIPLE_HEAPS
17166 for (int i = 0; i < n_heaps; i++)
17168 gc_heap* hp = g_heaps[i];
17169 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
17171 gc_requested = TRUE;
17175 #else //MULTIPLE_HEAPS
17176 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
17177 gc_requested = TRUE;
17178 #endif //MULTIPLE_HEAPS
17182 #ifdef MULTIPLE_HEAPS
17183 for (int i = 0; i < n_heaps; i++)
17185 gc_heap* hp = g_heaps[i];
17186 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
17188 soh_full_gc_requested = TRUE;
17192 #else //MULTIPLE_HEAPS
17193 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
17194 soh_full_gc_requested = TRUE;
17195 #endif //MULTIPLE_HEAPS
17199 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
17201 soh_full_gc_requested = TRUE;
17204 no_gc_requested = !(soh_full_gc_requested || gc_requested);
17206 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
17208 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17212 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
17214 // Check to see if we have enough reserved space.
17215 #ifdef MULTIPLE_HEAPS
17216 for (int i = 0; i < n_heaps; i++)
17218 gc_heap* hp = g_heaps[i];
17219 if (!hp->find_loh_space_for_no_gc())
17221 loh_full_gc_requested = TRUE;
17225 #else //MULTIPLE_HEAPS
17226 if (!find_loh_space_for_no_gc())
17227 loh_full_gc_requested = TRUE;
17228 #endif //MULTIPLE_HEAPS
17230 // Check to see if we have committed space.
17231 if (!loh_full_gc_requested)
17233 #ifdef MULTIPLE_HEAPS
17234 for (int i = 0; i < n_heaps; i++)
17236 gc_heap* hp = g_heaps[i];
17237 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
17239 loh_full_gc_requested = TRUE;
17243 #else //MULTIPLE_HEAPS
17244 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
17245 loh_full_gc_requested = TRUE;
17246 #endif //MULTIPLE_HEAPS
17250 if (loh_full_gc_requested || soh_full_gc_requested)
17252 if (current_no_gc_region_info.minimal_gc_p)
17253 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17256 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
17258 if (current_no_gc_region_info.start_status == start_no_gc_success)
17260 if (no_gc_requested)
17261 set_allocations_for_no_gc();
17266 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
17270 // We are done with starting the no_gc_region.
17271 current_no_gc_region_info.started = TRUE;
17276 end_no_gc_region_status gc_heap::end_no_gc_region()
17278 dprintf (1, ("end no gc called"));
17280 end_no_gc_region_status status = end_no_gc_success;
17282 if (!(current_no_gc_region_info.started))
17283 status = end_no_gc_not_in_progress;
17284 if (current_no_gc_region_info.num_gcs_induced)
17285 status = end_no_gc_induced;
17286 else if (current_no_gc_region_info.num_gcs)
17287 status = end_no_gc_alloc_exceeded;
17289 if (settings.pause_mode == pause_no_gc)
17290 restore_data_for_no_gc();
17292 // sets current_no_gc_region_info.started to FALSE here.
17293 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
17299 void gc_heap::update_collection_counts ()
17301 dynamic_data* dd0 = dynamic_data_of (0);
17302 dd_gc_clock (dd0) += 1;
17304 uint64_t now = GetHighPrecisionTimeStamp();
17306 for (int i = 0; i <= settings.condemned_generation;i++)
17308 dynamic_data* dd = dynamic_data_of (i);
17309 dd_collection_count (dd)++;
17310 //this is needed by the linear allocation model
17311 if (i == max_generation)
17313 dd_collection_count (dynamic_data_of (loh_generation))++;
17314 dd_collection_count(dynamic_data_of(poh_generation))++;
17317 dd_gc_clock (dd) = dd_gc_clock (dd0);
17318 dd_time_clock (dd) = now;
17322 BOOL gc_heap::expand_soh_with_minimal_gc()
17324 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
17327 heap_segment* new_seg = soh_get_segment_to_expand();
17330 if (g_gc_card_table != card_table)
17331 copy_brick_card_table();
17333 settings.promotion = TRUE;
17334 settings.demotion = FALSE;
17335 ephemeral_promotion = TRUE;
17336 int condemned_gen_number = max_generation - 1;
17338 int align_const = get_alignment_constant (TRUE);
17340 for (int i = 0; i <= condemned_gen_number; i++)
17342 generation* gen = generation_of (i);
17343 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
17344 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
17347 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
17348 // and need to make sure that there are no left over bricks from the previous GCs for the space
17349 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
17350 // ephemeral GCs later.
17351 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17352 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
17358 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
17359 generation_allocation_start (generation_of (max_generation - 1)));
17360 heap_segment_next (ephemeral_heap_segment) = new_seg;
17361 ephemeral_heap_segment = new_seg;
17362 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
17364 for (int i = condemned_gen_number; i >= 0; i--)
17366 size_t gen_start_size = Align (min_obj_size);
17367 make_generation (i, ephemeral_heap_segment, start);
17369 generation* gen = generation_of (i);
17370 generation_plan_allocation_start (gen) = start;
17371 generation_plan_allocation_start_size (gen) = gen_start_size;
17372 start += gen_start_size;
17374 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
17375 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
17377 fix_generation_bounds (condemned_gen_number, generation_of (0));
17379 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
17380 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
17382 adjust_ephemeral_limits();
17391 // Only to be done on the thread that calls restart in a join for server GC
17392 // and reset the oom status per heap.
17393 void gc_heap::check_and_set_no_gc_oom()
17395 #ifdef MULTIPLE_HEAPS
17396 for (int i = 0; i < n_heaps; i++)
17398 gc_heap* hp = g_heaps[i];
17399 if (hp->no_gc_oom_p)
17401 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17402 hp->no_gc_oom_p = false;
17408 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17409 no_gc_oom_p = false;
17411 #endif //MULTIPLE_HEAPS
17414 void gc_heap::allocate_for_no_gc_after_gc()
17416 if (current_no_gc_region_info.minimal_gc_p)
17417 repair_allocation_contexts (TRUE);
17419 no_gc_oom_p = false;
17421 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
17423 if (current_no_gc_region_info.soh_allocation_size != 0)
17425 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
17426 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
17428 no_gc_oom_p = true;
17431 #ifdef MULTIPLE_HEAPS
17432 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
17433 if (gc_t_join.joined())
17434 #endif //MULTIPLE_HEAPS
17436 check_and_set_no_gc_oom();
17438 #ifdef MULTIPLE_HEAPS
17439 gc_t_join.restart();
17440 #endif //MULTIPLE_HEAPS
17444 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
17445 !(current_no_gc_region_info.minimal_gc_p) &&
17446 (current_no_gc_region_info.loh_allocation_size != 0))
17448 gc_policy = policy_compact;
17449 saved_loh_segment_no_gc = 0;
17451 if (!find_loh_free_for_no_gc())
17453 heap_segment* seg = generation_allocation_segment (generation_of (loh_generation));
17454 BOOL found_seg_p = FALSE;
17457 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
17459 found_seg_p = TRUE;
17460 if (!commit_loh_for_no_gc (seg))
17462 no_gc_oom_p = true;
17466 seg = heap_segment_next (seg);
17470 gc_policy = policy_expand;
17473 #ifdef MULTIPLE_HEAPS
17474 gc_t_join.join(this, gc_join_expand_loh_no_gc);
17475 if (gc_t_join.joined())
17477 check_and_set_no_gc_oom();
17479 if (current_no_gc_region_info.start_status == start_no_gc_success)
17481 for (int i = 0; i < n_heaps; i++)
17483 gc_heap* hp = g_heaps[i];
17484 if (hp->gc_policy == policy_expand)
17486 hp->saved_loh_segment_no_gc = get_segment_for_uoh (loh_generation, get_uoh_seg_size (loh_allocation_no_gc), hp);
17487 if (!(hp->saved_loh_segment_no_gc))
17489 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17496 gc_t_join.restart();
17498 #else //MULTIPLE_HEAPS
17499 check_and_set_no_gc_oom();
17501 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
17503 saved_loh_segment_no_gc = get_segment_for_uoh (loh_generation, get_uoh_seg_size (loh_allocation_no_gc));
17504 if (!saved_loh_segment_no_gc)
17505 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17507 #endif //MULTIPLE_HEAPS
17509 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
17511 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
17513 no_gc_oom_p = true;
17519 #ifdef MULTIPLE_HEAPS
17520 gc_t_join.join(this, gc_join_final_no_gc);
17521 if (gc_t_join.joined())
17522 #endif //MULTIPLE_HEAPS
17524 check_and_set_no_gc_oom();
17526 if (current_no_gc_region_info.start_status == start_no_gc_success)
17528 set_allocations_for_no_gc();
17529 current_no_gc_region_info.started = TRUE;
17532 #ifdef MULTIPLE_HEAPS
17533 gc_t_join.restart();
17534 #endif //MULTIPLE_HEAPS
17538 void gc_heap::init_records()
17540 // An option is to move this to be after we figure out which gen to condemn so we don't
17541 // need to clear some generations' data 'cause we know they don't change, but that also means
17542 // we can't simply call memset here.
17543 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
17544 gc_data_per_heap.heap_index = heap_number;
17545 if (heap_number == 0)
17546 memset (&gc_data_global, 0, sizeof (gc_data_global));
17548 #ifdef GC_CONFIG_DRIVEN
17549 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
17550 #endif //GC_CONFIG_DRIVEN
17551 memset (&fgm_result, 0, sizeof (fgm_result));
17553 for (int i = 0; i < total_generation_count; i++)
17555 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
17556 generation* gen = generation_of (i);
17557 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
17558 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
17561 sufficient_gen0_space_p = FALSE;
17563 #ifdef MULTIPLE_HEAPS
17564 gen0_allocated_after_gc_p = false;
17565 #endif //MULTIPLE_HEAPS
17567 #if defined (_DEBUG) && defined (VERIFY_HEAP)
17568 verify_pinned_queue_p = FALSE;
17569 #endif // _DEBUG && VERIFY_HEAP
17572 void gc_heap::pm_full_gc_init_or_clear()
17574 // This means the next GC will be a full blocking GC and we need to init.
17575 if (settings.condemned_generation == (max_generation - 1))
17577 if (pm_trigger_full_gc)
17579 #ifdef MULTIPLE_HEAPS
17581 #endif //MULTIPLE_HEAPS
17582 dprintf (GTC_LOG, ("init for PM triggered full GC"));
17583 uint32_t saved_entry_memory_load = settings.entry_memory_load;
17584 settings.init_mechanisms();
17585 settings.reason = reason_pm_full_gc;
17586 settings.condemned_generation = max_generation;
17587 settings.entry_memory_load = saved_entry_memory_load;
17588 // Can't assert this since we only check at the end of gen2 GCs,
17589 // during gen1 the memory load could have already dropped.
17590 // Although arguably we should just turn off PM then...
17591 //assert (settings.entry_memory_load >= high_memory_load_th);
17592 assert (settings.entry_memory_load > 0);
17593 settings.gc_index += 1;
17597 // This means we are in the progress of a full blocking GC triggered by
17599 else if (settings.reason == reason_pm_full_gc)
17601 assert (settings.condemned_generation == max_generation);
17602 assert (pm_trigger_full_gc);
17603 pm_trigger_full_gc = false;
17605 dprintf (GTC_LOG, ("PM triggered full GC done"));
17609 void gc_heap::garbage_collect_pm_full_gc()
17611 assert (settings.condemned_generation == max_generation);
17612 assert (settings.reason == reason_pm_full_gc);
17613 assert (!settings.concurrent);
17617 void gc_heap::garbage_collect (int n)
17619 //reset the number of alloc contexts
17620 alloc_contexts_used = 0;
17622 fix_allocation_contexts (TRUE);
17623 #ifdef MULTIPLE_HEAPS
17625 gc_t_join.start_ts(this);
17626 #endif //JOIN_STATS
17627 clear_gen0_bricks();
17628 #endif //MULTIPLE_HEAPS
17630 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
17632 #ifdef MULTIPLE_HEAPS
17633 gc_t_join.join(this, gc_join_minimal_gc);
17634 if (gc_t_join.joined())
17635 #endif //MULTIPLE_HEAPS
17637 #ifdef MULTIPLE_HEAPS
17638 // this is serialized because we need to get a segment
17639 for (int i = 0; i < n_heaps; i++)
17641 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
17642 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17645 if (!expand_soh_with_minimal_gc())
17646 current_no_gc_region_info.start_status = start_no_gc_no_memory;
17647 #endif //MULTIPLE_HEAPS
17649 update_collection_counts_for_no_gc();
17651 #ifdef MULTIPLE_HEAPS
17652 gc_t_join.restart();
17653 #endif //MULTIPLE_HEAPS
17661 settings.reason = gc_trigger_reason;
17662 num_pinned_objects = 0;
17665 if (settings.reason == reason_gcstress)
17667 settings.reason = reason_induced;
17668 settings.stress_induced = TRUE;
17670 #endif // STRESS_HEAP
17672 #ifdef MULTIPLE_HEAPS
17673 //align all heaps on the max generation to condemn
17674 dprintf (3, ("Joining for max generation to condemn"));
17675 condemned_generation_num = generation_to_condemn (n,
17676 &blocking_collection,
17677 &elevation_requested,
17679 gc_t_join.join(this, gc_join_generation_determined);
17680 if (gc_t_join.joined())
17681 #endif //MULTIPLE_HEAPS
17683 #ifdef FEATURE_BASICFREEZE
17684 seg_table->delete_old_slots();
17685 #endif //FEATURE_BASICFREEZE
17687 #ifdef MULTIPLE_HEAPS
17688 for (int i = 0; i < n_heaps; i++)
17690 gc_heap* hp = g_heaps[i];
17691 // check for card table growth
17692 if (g_gc_card_table != hp->card_table)
17693 hp->copy_brick_card_table();
17695 hp->rearrange_uoh_segments();
17696 #ifdef BACKGROUND_GC
17697 hp->background_delay_delete_uoh_segments();
17698 if (!gc_heap::background_running_p())
17699 hp->rearrange_small_heap_segments();
17700 #endif //BACKGROUND_GC
17702 #else //MULTIPLE_HEAPS
17703 if (g_gc_card_table != card_table)
17704 copy_brick_card_table();
17706 rearrange_uoh_segments();
17707 #ifdef BACKGROUND_GC
17708 background_delay_delete_uoh_segments();
17709 if (!gc_heap::background_running_p())
17710 rearrange_small_heap_segments();
17711 #endif //BACKGROUND_GC
17712 #endif //MULTIPLE_HEAPS
17714 BOOL should_evaluate_elevation = TRUE;
17715 BOOL should_do_blocking_collection = FALSE;
17717 #ifdef MULTIPLE_HEAPS
17718 int gen_max = condemned_generation_num;
17719 for (int i = 0; i < n_heaps; i++)
17721 if (gen_max < g_heaps[i]->condemned_generation_num)
17722 gen_max = g_heaps[i]->condemned_generation_num;
17723 if (should_evaluate_elevation && !(g_heaps[i]->elevation_requested))
17724 should_evaluate_elevation = FALSE;
17725 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
17726 should_do_blocking_collection = TRUE;
17729 settings.condemned_generation = gen_max;
17730 #else //MULTIPLE_HEAPS
17731 settings.condemned_generation = generation_to_condemn (n,
17732 &blocking_collection,
17733 &elevation_requested,
17735 should_evaluate_elevation = elevation_requested;
17736 should_do_blocking_collection = blocking_collection;
17737 #endif //MULTIPLE_HEAPS
17739 settings.condemned_generation = joined_generation_to_condemn (
17740 should_evaluate_elevation,
17742 settings.condemned_generation,
17743 &should_do_blocking_collection
17747 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
17748 "condemned generation num: %d\n", settings.condemned_generation);
17750 record_gcs_during_no_gc();
17752 if (settings.condemned_generation > 1)
17753 settings.promotion = TRUE;
17755 #ifdef HEAP_ANALYZE
17756 // At this point we've decided what generation is condemned
17757 // See if we've been requested to analyze survivors after the mark phase
17758 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
17760 heap_analyze_enabled = TRUE;
17762 #endif // HEAP_ANALYZE
17764 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
17766 #ifdef BACKGROUND_GC
17767 if ((settings.condemned_generation == max_generation) &&
17768 (gc_heap::background_running_p()))
17770 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
17771 // because we have to collect 0 and 1 properly
17772 // in particular, the allocation contexts are gone.
17773 // For now, it is simpler to collect max_generation-1
17774 settings.condemned_generation = max_generation - 1;
17775 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
17778 if ((settings.condemned_generation == max_generation) &&
17779 (should_do_blocking_collection == FALSE) &&
17780 gc_can_use_concurrent &&
17781 !temp_disable_concurrent_p &&
17782 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
17784 keep_bgc_threads_p = TRUE;
17785 c_write (settings.concurrent, TRUE);
17786 memset (&bgc_data_global, 0, sizeof(bgc_data_global));
17787 memcpy (&bgc_data_global, &gc_data_global, sizeof(gc_data_global));
17790 #endif //BACKGROUND_GC
17792 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
17794 #ifdef MULTIPLE_HEAPS
17795 hb_log_balance_activities();
17796 hb_log_new_allocation();
17797 #endif //MULTIPLE_HEAPS
17799 // Call the EE for start of GC work
17800 GCToEEInterface::GcStartWork (settings.condemned_generation,
17803 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
17804 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
17805 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
17809 #ifdef MULTIPLE_HEAPS
17810 gc_start_event.Reset();
17811 dprintf(3, ("Starting all gc threads for gc"));
17812 gc_t_join.restart();
17813 #endif //MULTIPLE_HEAPS
17816 descr_generations (TRUE);
17819 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
17820 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
17822 verify_heap (TRUE);
17824 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
17825 checkGCWriteBarrier();
17827 #endif // VERIFY_HEAP
17829 #ifdef BACKGROUND_GC
17830 if (settings.concurrent)
17832 // We need to save the settings because we'll need to restore it after each FGC.
17833 assert (settings.condemned_generation == max_generation);
17834 settings.compaction = FALSE;
17835 saved_bgc_settings = settings;
17837 #ifdef MULTIPLE_HEAPS
17838 if (heap_number == 0)
17840 for (int i = 0; i < n_heaps; i++)
17842 prepare_bgc_thread (g_heaps[i]);
17844 dprintf (2, ("setting bgc_threads_sync_event"));
17845 bgc_threads_sync_event.Set();
17849 bgc_threads_sync_event.Wait(INFINITE, FALSE);
17850 dprintf (2, ("bgc_threads_sync_event is signalled"));
17853 prepare_bgc_thread(0);
17854 #endif //MULTIPLE_HEAPS
17856 #ifdef MULTIPLE_HEAPS
17857 gc_t_join.join(this, gc_join_start_bgc);
17858 if (gc_t_join.joined())
17859 #endif //MULTIPLE_HEAPS
17861 do_concurrent_p = TRUE;
17862 do_ephemeral_gc_p = FALSE;
17863 #ifdef MULTIPLE_HEAPS
17864 dprintf(2, ("Joined to perform a background GC"));
17866 for (int i = 0; i < n_heaps; i++)
17868 gc_heap* hp = g_heaps[i];
17869 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init())
17871 do_concurrent_p = FALSE;
17876 hp->background_saved_lowest_address = hp->lowest_address;
17877 hp->background_saved_highest_address = hp->highest_address;
17881 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init());
17882 if (do_concurrent_p)
17884 background_saved_lowest_address = lowest_address;
17885 background_saved_highest_address = highest_address;
17887 #endif //MULTIPLE_HEAPS
17889 if (do_concurrent_p)
17891 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17892 SoftwareWriteWatch::EnableForGCHeap();
17893 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17895 #ifdef MULTIPLE_HEAPS
17896 for (int i = 0; i < n_heaps; i++)
17897 g_heaps[i]->current_bgc_state = bgc_initialized;
17899 current_bgc_state = bgc_initialized;
17900 #endif //MULTIPLE_HEAPS
17902 int gen = check_for_ephemeral_alloc();
17903 // always do a gen1 GC before we start BGC.
17904 dont_restart_ee_p = TRUE;
17907 // If we decide to not do a GC before the BGC we need to
17908 // restore the gen0 alloc context.
17909 #ifdef MULTIPLE_HEAPS
17910 for (int i = 0; i < n_heaps; i++)
17912 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
17913 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
17916 generation_allocation_pointer (youngest_generation) = 0;
17917 generation_allocation_limit (youngest_generation) = 0;
17918 #endif //MULTIPLE_HEAPS
17922 do_ephemeral_gc_p = TRUE;
17924 settings.init_mechanisms();
17925 settings.condemned_generation = gen;
17926 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
17929 // TODO BACKGROUND_GC need to add the profiling stuff here.
17930 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
17933 //clear the cards so they don't bleed in gen 1 during collection
17934 // shouldn't this always be done at the beginning of any GC?
17935 //clear_card_for_addresses (
17936 // generation_allocation_start (generation_of (0)),
17937 // heap_segment_allocated (ephemeral_heap_segment));
17939 if (!do_ephemeral_gc_p)
17941 do_background_gc();
17946 settings.compaction = TRUE;
17947 c_write (settings.concurrent, FALSE);
17950 #ifdef MULTIPLE_HEAPS
17951 gc_t_join.restart();
17952 #endif //MULTIPLE_HEAPS
17955 if (do_concurrent_p)
17957 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
17958 // global data is only calculated at the end of the GC so we don't need to worry about
17959 // FGCs overwriting it.
17960 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
17961 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
17963 if (do_ephemeral_gc_p)
17965 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
17967 gen_to_condemn_reasons.init();
17968 gen_to_condemn_reasons.set_condition (gen_before_bgc);
17969 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
17971 #ifdef MULTIPLE_HEAPS
17972 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
17973 if (gc_t_join.joined())
17974 #endif //MULTIPLE_HEAPS
17976 #ifdef MULTIPLE_HEAPS
17978 #endif //MULTIPLE_HEAPS
17979 settings = saved_bgc_settings;
17980 assert (settings.concurrent);
17982 do_background_gc();
17984 #ifdef MULTIPLE_HEAPS
17985 gc_t_join.restart();
17986 #endif //MULTIPLE_HEAPS
17992 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
17997 #endif //BACKGROUND_GC
18001 #ifndef MULTIPLE_HEAPS
18002 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
18003 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
18004 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
18005 #endif //MULTIPLE_HEAPS
18008 if (settings.pause_mode == pause_no_gc)
18009 allocate_for_no_gc_after_gc();
18012 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
18015 size_t& gc_heap::promoted_bytes(int thread)
18017 #ifdef MULTIPLE_HEAPS
18018 return g_promoted [thread*16];
18019 #else //MULTIPLE_HEAPS
18020 UNREFERENCED_PARAMETER(thread);
18022 #endif //MULTIPLE_HEAPS
18025 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
18027 heap_segment* seg = seg_mapping_table_segment_of (interior);
18030 if (small_segment_only_p && heap_segment_uoh_p (seg))
18036 #if !defined(_DEBUG) && !defined(__GNUC__)
18037 inline // This causes link errors if global optimization is off
18038 #endif //!_DEBUG && !__GNUC__
18039 gc_heap* gc_heap::heap_of (uint8_t* o)
18041 #ifdef MULTIPLE_HEAPS
18043 return g_heaps [0];
18044 gc_heap* hp = seg_mapping_table_heap_of (o);
18045 return (hp ? hp : g_heaps[0]);
18046 #else //MULTIPLE_HEAPS
18047 UNREFERENCED_PARAMETER(o);
18049 #endif //MULTIPLE_HEAPS
18053 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
18055 #ifdef MULTIPLE_HEAPS
18057 return g_heaps [0];
18058 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
18059 return (hp ? hp : g_heaps[0]);
18060 #else //MULTIPLE_HEAPS
18061 UNREFERENCED_PARAMETER(o);
18063 #endif //MULTIPLE_HEAPS
18066 // will find all heap objects (large and small)
18068 // Callers of this method need to guarantee the interior pointer is within the heap range.
18070 // If you need it to be stricter, eg if you only want to find an object in ephemeral range,
18071 // you should make sure interior is within that range before calling this method.
18072 uint8_t* gc_heap::find_object (uint8_t* interior)
18074 assert (interior != 0);
18076 if (!gen0_bricks_cleared)
18078 #ifdef MULTIPLE_HEAPS
18079 assert (!"Should have already been done in server GC");
18080 #endif //MULTIPLE_HEAPS
18081 clear_gen0_bricks();
18083 //indicate that in the future this needs to be done during allocation
18084 gen0_must_clear_bricks = FFIND_DECAY;
18086 int brick_entry = get_brick_entry(brick_of (interior));
18087 if (brick_entry == 0)
18089 // this is a pointer to a UOH object
18090 heap_segment* seg = find_segment (interior, FALSE);
18092 #ifdef FEATURE_CONSERVATIVE_GC
18093 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
18097 // If interior falls within the first free object at the beginning of a generation,
18098 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
18099 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
18100 #ifdef FEATURE_CONSERVATIVE_GC
18101 || (GCConfig::GetConservativeGC() && !heap_segment_uoh_p (seg))
18104 assert (interior < heap_segment_allocated (seg));
18106 uint8_t* o = heap_segment_mem (seg);
18107 while (o < heap_segment_allocated (seg))
18109 uint8_t* next_o = o + Align (size (o), align_const);
18110 assert (next_o > o);
18111 if ((o <= interior) && (interior < next_o))
18124 heap_segment* seg = find_segment (interior, TRUE);
18127 #ifdef FEATURE_CONSERVATIVE_GC
18128 if (interior >= heap_segment_allocated (seg))
18131 assert (interior < heap_segment_allocated (seg));
18133 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
18141 #ifdef MULTIPLE_HEAPS
18144 #ifdef GC_CONFIG_DRIVEN
18145 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;}}
18146 #else //GC_CONFIG_DRIVEN
18147 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}}
18148 #endif //GC_CONFIG_DRIVEN
18150 #define m_boundary(o) {}
18153 #define m_boundary_fullgc(o) {}
18155 #else //MULTIPLE_HEAPS
18158 #ifdef GC_CONFIG_DRIVEN
18159 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
18161 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
18162 #endif //GC_CONFIG_DRIVEN
18164 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
18167 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
18169 #endif //MULTIPLE_HEAPS
18171 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
18174 BOOL gc_heap::gc_mark1 (uint8_t* o)
18176 BOOL marked = !marked (o);
18178 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
18183 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
18185 BOOL marked = FALSE;
18186 if ((o >= low) && (o < high))
18187 marked = gc_mark1 (o);
18188 #ifdef MULTIPLE_HEAPS
18191 gc_heap* hp = heap_of_gc (o);
18193 if ((o >= hp->gc_low) && (o < hp->gc_high))
18194 marked = gc_mark1 (o);
18197 snoop_stat.objects_checked_count++;
18201 snoop_stat.objects_marked_count++;
18205 snoop_stat.zero_ref_count++;
18208 #endif //SNOOP_STATS
18209 #endif //MULTIPLE_HEAPS
18213 #ifdef BACKGROUND_GC
18216 BOOL gc_heap::background_marked (uint8_t* o)
18218 return mark_array_marked (o);
18221 BOOL gc_heap::background_mark1 (uint8_t* o)
18223 BOOL to_mark = !mark_array_marked (o);
18225 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
18228 mark_array_set_marked (o);
18229 dprintf (4, ("n*%Ix*n", (size_t)o));
18236 // TODO: we could consider filtering out NULL's here instead of going to
18237 // look for it on other heaps
18239 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
18241 BOOL marked = FALSE;
18242 if ((o >= low) && (o < high))
18243 marked = background_mark1 (o);
18244 #ifdef MULTIPLE_HEAPS
18247 gc_heap* hp = heap_of (o);
18249 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
18250 marked = background_mark1 (o);
18252 #endif //MULTIPLE_HEAPS
18256 #endif //BACKGROUND_GC
18258 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
18259 #define ignore_start 0
18260 #define use_start 1
18262 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
18264 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
18265 CGCDescSeries* cur = map->GetHighestSeries(); \
18266 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
18270 CGCDescSeries* last = map->GetLowestSeries(); \
18271 uint8_t** parm = 0; \
18274 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
18275 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
18276 uint8_t** ppstop = \
18277 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
18278 if (!start_useful || (uint8_t*)ppstop > (start)) \
18280 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
18281 while (parm < ppstop) \
18289 } while (cur >= last); \
18293 /* Handle the repeating case - array of valuetypes */ \
18294 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
18295 if (start_useful && start > (uint8_t*)parm) \
18297 ptrdiff_t cs = mt->RawGetComponentSize(); \
18298 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
18300 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
18302 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
18304 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
18305 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
18306 uint8_t** ppstop = parm + nptrs; \
18307 if (!start_useful || (uint8_t*)ppstop > (start)) \
18309 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
18314 } while (parm < ppstop); \
18316 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
18322 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
18324 // 1 thing to note about this macro:
18325 // 1) you can use *parm safely but in general you don't want to use parm
18326 // because for the collectible types it's not an address on the managed heap.
18327 #ifndef COLLECTIBLE_CLASS
18328 #define go_through_object_cl(mt,o,size,parm,exp) \
18330 if (header(o)->ContainsPointers()) \
18332 go_through_object_nostart(mt,o,size,parm,exp); \
18335 #else //COLLECTIBLE_CLASS
18336 #define go_through_object_cl(mt,o,size,parm,exp) \
18338 if (header(o)->Collectible()) \
18340 uint8_t* class_obj = get_class_object (o); \
18341 uint8_t** parm = &class_obj; \
18342 do {exp} while (false); \
18344 if (header(o)->ContainsPointers()) \
18346 go_through_object_nostart(mt,o,size,parm,exp); \
18349 #endif //COLLECTIBLE_CLASS
18351 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
18352 void gc_heap::enque_pinned_plug (uint8_t* plug,
18353 BOOL save_pre_plug_info_p,
18354 uint8_t* last_object_in_last_plug)
18356 if (mark_stack_array_length <= mark_stack_tos)
18358 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
18360 // we don't want to continue here due to security
18361 // risks. This happens very rarely and fixing it in the
18362 // way so that we can continue is a bit involved and will
18363 // not be done in Dev10.
18364 GCToEEInterface::HandleFatalError((unsigned int)CORINFO_EXCEPTION_GC);
18368 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
18369 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
18370 mark& m = mark_stack_array[mark_stack_tos];
18372 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
18373 m.saved_pre_p = save_pre_plug_info_p;
18375 if (save_pre_plug_info_p)
18378 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
18380 clear_plug_padded (last_object_in_last_plug);
18381 #endif //SHORT_PLUGS
18382 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
18385 set_plug_padded (last_object_in_last_plug);
18386 #endif //SHORT_PLUGS
18388 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
18390 // If the last object in the last plug is too short, it requires special handling.
18391 size_t last_obj_size = plug - last_object_in_last_plug;
18392 if (last_obj_size < min_pre_pin_obj_size)
18394 record_interesting_data_point (idp_pre_short);
18397 record_interesting_data_point (idp_pre_short_padded);
18398 #endif //SHORT_PLUGS
18399 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
18400 last_object_in_last_plug, plug));
18401 // Need to set the short bit regardless of having refs or not because we need to
18402 // indicate that this object is not walkable.
18405 #ifdef COLLECTIBLE_CLASS
18406 if (is_collectible (last_object_in_last_plug))
18408 m.set_pre_short_collectible();
18410 #endif //COLLECTIBLE_CLASS
18412 if (contain_pointers (last_object_in_last_plug))
18414 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
18416 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
18418 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
18419 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
18420 m.set_pre_short_bit (gap_offset);
18427 m.saved_post_p = FALSE;
18430 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
18433 UNREFERENCED_PARAMETER(last_pinned_plug);
18436 mark& m = mark_stack_array[mark_stack_tos - 1];
18437 assert (last_pinned_plug == m.first);
18438 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
18441 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
18443 clear_plug_padded (last_object_in_last_plug);
18444 #endif //SHORT_PLUGS
18445 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
18448 set_plug_padded (last_object_in_last_plug);
18449 #endif //SHORT_PLUGS
18451 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
18453 // This is important - we need to clear all bits here except the last one.
18454 m.saved_post_p = TRUE;
18457 m.saved_post_plug_debug.gap = 1;
18460 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
18462 size_t last_obj_size = post_plug - last_object_in_last_plug;
18463 if (last_obj_size < min_pre_pin_obj_size)
18465 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
18466 record_interesting_data_point (idp_post_short);
18469 record_interesting_data_point (idp_post_short_padded);
18470 #endif //SHORT_PLUGS
18471 m.set_post_short();
18472 #if defined (_DEBUG) && defined (VERIFY_HEAP)
18473 verify_pinned_queue_p = TRUE;
18474 #endif // _DEBUG && VERIFY_HEAP
18476 #ifdef COLLECTIBLE_CLASS
18477 if (is_collectible (last_object_in_last_plug))
18479 m.set_post_short_collectible();
18481 #endif //COLLECTIBLE_CLASS
18483 if (contain_pointers (last_object_in_last_plug))
18485 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
18487 // TODO: since we won't be able to walk this object in relocation, we still need to
18488 // take care of collectible assemblies here.
18489 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
18491 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
18492 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
18493 m.set_post_short_bit (gap_offset);
18502 __declspec(naked) void __fastcall Prefetch(void* addr)
18510 inline void Prefetch (void* addr)
18512 UNREFERENCED_PARAMETER(addr);
18517 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
18519 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
18522 #endif //MH_SC_MARK
18526 #define partial_object 3
18528 uint8_t* ref_from_slot (uint8_t* r)
18530 return (uint8_t*)((size_t)r & ~(stolen | partial));
18533 BOOL stolen_p (uint8_t* r)
18535 return (((size_t)r&2) && !((size_t)r&1));
18538 BOOL ready_p (uint8_t* r)
18540 return ((size_t)r != 1);
18543 BOOL partial_p (uint8_t* r)
18545 return (((size_t)r&1) && !((size_t)r&2));
18548 BOOL straight_ref_p (uint8_t* r)
18550 return (!stolen_p (r) && !partial_p (r));
18553 BOOL partial_object_p (uint8_t* r)
18555 return (((size_t)r & partial_object) == partial_object);
18558 BOOL ref_p (uint8_t* r)
18560 return (straight_ref_p (r) || partial_object_p (r));
18563 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
18565 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
18566 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
18567 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
18568 #ifdef SORT_MARK_STACK
18569 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
18570 #endif //SORT_MARK_STACK
18572 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
18573 // update mark list.
18574 BOOL full_p = (settings.condemned_generation == max_generation);
18576 assert ((start >= oo) && (start < oo+size(oo)));
18579 *mark_stack_tos = oo;
18580 #endif //!MH_SC_MARK
18584 #ifdef MULTIPLE_HEAPS
18585 #else //MULTIPLE_HEAPS
18586 const int thread = 0;
18587 #endif //MULTIPLE_HEAPS
18589 if (oo && ((size_t)oo != 4))
18597 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18599 BOOL overflow_p = FALSE;
18601 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18603 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18604 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
18610 if (overflow_p == FALSE)
18612 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18614 go_through_object_cl (method_table(oo), oo, s, ppslot,
18616 uint8_t* o = *ppslot;
18618 if (gc_mark (o, gc_low, gc_high))
18622 m_boundary_fullgc (o);
18628 size_t obj_size = size (o);
18629 promoted_bytes (thread) += obj_size;
18630 if (contain_pointers_or_collectible (o))
18632 *(mark_stack_tos++) = o;
18640 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18641 min_overflow_address = min (min_overflow_address, oo);
18642 max_overflow_address = max (max_overflow_address, oo);
18647 if (partial_p (oo))
18649 start = ref_from_slot (oo);
18650 oo = ref_from_slot (*(--mark_stack_tos));
18651 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18652 assert ((oo < start) && (start < (oo + size (oo))));
18654 #ifdef COLLECTIBLE_CLASS
18657 // If there's a class object, push it now. We are guaranteed to have the slot since
18658 // we just popped one object off.
18659 if (is_collectible (oo))
18661 uint8_t* class_obj = get_class_object (oo);
18662 if (gc_mark (class_obj, gc_low, gc_high))
18666 m_boundary_fullgc (class_obj);
18670 m_boundary (class_obj);
18673 size_t obj_size = size (class_obj);
18674 promoted_bytes (thread) += obj_size;
18675 *(mark_stack_tos++) = class_obj;
18676 // The code below expects that the oo is still stored in the stack slot that was
18677 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
18678 // But the class_obj has just overwritten that stack slot and so the oo needs to
18679 // be stored to the new slot that's pointed to by the mark_stack_tos.
18680 *mark_stack_tos = oo;
18684 if (!contain_pointers (oo))
18689 #endif //COLLECTIBLE_CLASS
18693 BOOL overflow_p = FALSE;
18695 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18699 if (overflow_p == FALSE)
18701 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18703 //push the object and its current
18704 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
18708 *(place) = (uint8_t*)partial;
18709 #endif //MH_SC_MARK
18710 int i = num_partial_refs;
18711 uint8_t* ref_to_continue = 0;
18713 go_through_object (method_table(oo), oo, s, ppslot,
18714 start, use_start, (oo + s),
18716 uint8_t* o = *ppslot;
18718 if (gc_mark (o, gc_low, gc_high))
18722 m_boundary_fullgc (o);
18728 size_t obj_size = size (o);
18729 promoted_bytes (thread) += obj_size;
18730 if (contain_pointers_or_collectible (o))
18732 *(mark_stack_tos++) = o;
18735 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
18744 //we are finished with this object
18745 assert (ref_to_continue == 0);
18747 assert ((*(place-1)) == (uint8_t*)0);
18750 #endif //MH_SC_MARK
18752 // shouldn't we decrease tos by 2 here??
18755 if (ref_to_continue)
18759 assert ((*(place-1)) == (uint8_t*)0);
18760 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
18761 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
18762 #endif //MH_SC_MARK
18763 *place = ref_to_continue;
18768 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18769 min_overflow_address = min (min_overflow_address, oo);
18770 max_overflow_address = max (max_overflow_address, oo);
18773 #ifdef SORT_MARK_STACK
18774 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18776 rqsort1 (sorted_tos, mark_stack_tos-1);
18777 sorted_tos = mark_stack_tos-1;
18779 #endif //SORT_MARK_STACK
18782 if (!(mark_stack_empty_p()))
18784 oo = *(--mark_stack_tos);
18787 #ifdef SORT_MARK_STACK
18788 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
18789 #endif //SORT_MARK_STACK
18797 BOOL same_numa_node_p (int hn1, int hn2)
18799 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
18802 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
18804 int hn = (current_buddy+1)%n_heaps;
18805 while (hn != current_buddy)
18807 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
18809 hn = (hn+1)%n_heaps;
18811 return current_buddy;
18815 gc_heap::mark_steal()
18817 mark_stack_busy() = 0;
18818 //clear the mark stack in the snooping range
18819 for (int i = 0; i < max_snoop_level; i++)
18821 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18824 //pick the next heap as our buddy
18825 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
18828 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
18829 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18830 #endif //SNOOP_STATS
18832 int idle_loop_count = 0;
18833 int first_not_ready_level = 0;
18837 gc_heap* hp = g_heaps [thpn];
18838 int level = first_not_ready_level;
18839 first_not_ready_level = 0;
18841 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
18843 idle_loop_count = 0;
18845 snoop_stat.busy_count++;
18846 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
18847 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
18848 #endif //SNOOP_STATS
18850 uint8_t* o = ref_mark_stack (hp, level);
18852 uint8_t* start = o;
18855 mark_stack_busy() = 1;
18857 BOOL success = TRUE;
18858 uint8_t* next = (ref_mark_stack (hp, level+1));
18861 if (((size_t)o > 4) && !partial_object_p (o))
18863 //this is a normal object, not a partial mark tuple
18864 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
18865 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
18867 snoop_stat.interlocked_count++;
18869 snoop_stat.normal_count++;
18870 #endif //SNOOP_STATS
18874 //it is a stolen entry, or beginning/ending of a partial mark
18877 snoop_stat.stolen_or_pm_count++;
18878 #endif //SNOOP_STATS
18882 else if (stolen_p (next))
18884 //ignore the stolen guy and go to the next level
18888 snoop_stat.stolen_entry_count++;
18889 #endif //SNOOP_STATS
18893 assert (partial_p (next));
18894 start = ref_from_slot (next);
18895 //re-read the object
18896 o = ref_from_slot (ref_mark_stack (hp, level));
18900 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
18902 snoop_stat.interlocked_count++;
18905 snoop_stat.partial_mark_parent_count++;
18907 #endif //SNOOP_STATS
18911 // stack is not ready, or o is completely different from the last time we read from this stack level.
18912 // go up 2 levels to steal children or totally unrelated objects.
18914 if (first_not_ready_level == 0)
18916 first_not_ready_level = level;
18920 snoop_stat.pm_not_ready_count++;
18921 #endif //SNOOP_STATS
18928 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18929 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18930 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18931 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18932 #endif //SNOOP_STATS
18934 mark_object_simple1 (o, start, heap_number);
18937 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18938 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18939 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18940 #endif //SNOOP_STATS
18942 mark_stack_busy() = 0;
18944 //clear the mark stack in snooping range
18945 for (int i = 0; i < max_snoop_level; i++)
18947 if (((uint8_t**)mark_stack_array)[i] != 0)
18949 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18951 snoop_stat.stack_bottom_clear_count++;
18952 #endif //SNOOP_STATS
18958 mark_stack_busy() = 0;
18962 //slot is either partial or stolen
18966 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18970 if (!hp->mark_stack_busy())
18972 first_not_ready_level = 0;
18975 if ((idle_loop_count % (6) )==1)
18978 snoop_stat.switch_to_thread_count++;
18979 #endif //SNOOP_STATS
18980 GCToOSInterface::Sleep(1);
18982 int free_count = 1;
18984 snoop_stat.stack_idle_count++;
18985 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18986 #endif //SNOOP_STATS
18987 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18989 if (!((g_heaps [hpn])->mark_stack_busy()))
18993 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18994 #endif //SNOOP_STATS
18996 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
19001 hpn = (hpn+1)%n_heaps;
19004 if (free_count == n_heaps)
19013 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
19016 snoop_stat.check_level_count++;
19017 #endif //SNOOP_STATS
19018 return (next_heap->mark_stack_busy()>=1);
19020 #endif //MH_SC_MARK
19023 void gc_heap::print_snoop_stat()
19025 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
19026 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
19027 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
19028 snoop_stat.heap_index,
19029 snoop_stat.objects_checked_count,
19030 snoop_stat.zero_ref_count,
19031 snoop_stat.objects_marked_count,
19032 snoop_stat.stolen_stack_count,
19033 snoop_stat.partial_stack_count,
19034 snoop_stat.normal_stack_count,
19035 snoop_stat.non_stack_count));
19036 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
19037 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
19038 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19039 snoop_stat.heap_index,
19040 snoop_stat.check_level_count,
19041 snoop_stat.busy_count,
19042 snoop_stat.interlocked_count,
19043 snoop_stat.partial_mark_parent_count,
19044 snoop_stat.stolen_or_pm_count,
19045 snoop_stat.stolen_entry_count,
19046 snoop_stat.pm_not_ready_count,
19047 snoop_stat.normal_count,
19048 snoop_stat.stack_bottom_clear_count));
19050 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
19051 "heap", "check", "zero", "mark", "idle", "switch");
19052 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
19053 snoop_stat.heap_index,
19054 snoop_stat.objects_checked_count,
19055 snoop_stat.zero_ref_count,
19056 snoop_stat.objects_marked_count,
19057 snoop_stat.stack_idle_count,
19058 snoop_stat.switch_to_thread_count);
19059 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
19060 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
19061 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19062 snoop_stat.heap_index,
19063 snoop_stat.check_level_count,
19064 snoop_stat.busy_count,
19065 snoop_stat.interlocked_count,
19066 snoop_stat.partial_mark_parent_count,
19067 snoop_stat.stolen_or_pm_count,
19068 snoop_stat.stolen_entry_count,
19069 snoop_stat.pm_not_ready_count,
19070 snoop_stat.normal_count,
19071 snoop_stat.stack_bottom_clear_count);
19073 #endif //SNOOP_STATS
19075 #ifdef HEAP_ANALYZE
19077 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
19079 if (!internal_root_array)
19081 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
19082 if (!internal_root_array)
19084 heap_analyze_success = FALSE;
19088 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
19090 size_t new_size = 2*internal_root_array_length;
19092 uint64_t available_physical = 0;
19093 get_memory_info (NULL, &available_physical);
19094 if (new_size > (size_t)(available_physical / 10))
19096 heap_analyze_success = FALSE;
19100 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19103 memcpy (tmp, internal_root_array,
19104 internal_root_array_length*sizeof (uint8_t*));
19105 delete[] internal_root_array;
19106 internal_root_array = tmp;
19107 internal_root_array_length = new_size;
19111 heap_analyze_success = FALSE;
19116 if (heap_analyze_success)
19118 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
19120 uint8_t* ref = (uint8_t*)po;
19121 if (!current_obj ||
19122 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
19124 gc_heap* hp = gc_heap::heap_of (ref);
19125 current_obj = hp->find_object (ref);
19126 current_obj_size = size (current_obj);
19128 internal_root_array[internal_root_array_index] = current_obj;
19129 internal_root_array_index++;
19133 mark_object_simple (po THREAD_NUMBER_ARG);
19135 #endif //HEAP_ANALYZE
19137 //this method assumes that *po is in the [low. high[ range
19139 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
19142 #ifdef MULTIPLE_HEAPS
19143 #else //MULTIPLE_HEAPS
19144 const int thread = 0;
19145 #endif //MULTIPLE_HEAPS
19148 snoop_stat.objects_checked_count++;
19149 #endif //SNOOP_STATS
19154 size_t s = size (o);
19155 promoted_bytes (thread) += s;
19157 go_through_object_cl (method_table(o), o, s, poo,
19159 uint8_t* oo = *poo;
19160 if (gc_mark (oo, gc_low, gc_high))
19163 size_t obj_size = size (oo);
19164 promoted_bytes (thread) += obj_size;
19166 if (contain_pointers_or_collectible (oo))
19167 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
19177 void gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
19179 if ((o >= gc_low) && (o < gc_high))
19180 mark_object_simple (&o THREAD_NUMBER_ARG);
19181 #ifdef MULTIPLE_HEAPS
19184 gc_heap* hp = heap_of (o);
19186 if ((o >= hp->gc_low) && (o < hp->gc_high))
19187 mark_object_simple (&o THREAD_NUMBER_ARG);
19189 #endif //MULTIPLE_HEAPS
19192 #ifdef BACKGROUND_GC
19194 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
19196 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
19198 #ifdef SORT_MARK_STACK
19199 uint8_t** sorted_tos = background_mark_stack_array;
19200 #endif //SORT_MARK_STACK
19202 background_mark_stack_tos = background_mark_stack_array;
19206 #ifdef MULTIPLE_HEAPS
19207 #else //MULTIPLE_HEAPS
19208 const int thread = 0;
19209 #endif //MULTIPLE_HEAPS
19213 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
19215 BOOL overflow_p = FALSE;
19217 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
19219 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
19220 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
19221 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
19223 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
19225 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
19229 bgc_overflow_count++;
19234 if (overflow_p == FALSE)
19236 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
19238 go_through_object_cl (method_table(oo), oo, s, ppslot,
19240 uint8_t* o = *ppslot;
19242 if (background_mark (o,
19243 background_saved_lowest_address,
19244 background_saved_highest_address))
19247 size_t obj_size = size (o);
19248 bpromoted_bytes (thread) += obj_size;
19249 if (contain_pointers_or_collectible (o))
19251 *(background_mark_stack_tos++) = o;
19260 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
19261 background_min_overflow_address = min (background_min_overflow_address, oo);
19262 background_max_overflow_address = max (background_max_overflow_address, oo);
19267 uint8_t* start = oo;
19268 if ((size_t)oo & 1)
19270 oo = (uint8_t*)((size_t)oo & ~1);
19271 start = *(--background_mark_stack_tos);
19272 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
19274 #ifdef COLLECTIBLE_CLASS
19277 // If there's a class object, push it now. We are guaranteed to have the slot since
19278 // we just popped one object off.
19279 if (is_collectible (oo))
19281 uint8_t* class_obj = get_class_object (oo);
19282 if (background_mark (class_obj,
19283 background_saved_lowest_address,
19284 background_saved_highest_address))
19286 size_t obj_size = size (class_obj);
19287 bpromoted_bytes (thread) += obj_size;
19289 *(background_mark_stack_tos++) = class_obj;
19293 if (!contain_pointers (oo))
19298 #endif //COLLECTIBLE_CLASS
19302 BOOL overflow_p = FALSE;
19304 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
19306 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
19307 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
19309 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
19311 (size_t)(mark_stack_limit - background_mark_stack_tos),
19317 bgc_overflow_count++;
19320 if (overflow_p == FALSE)
19322 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
19324 //push the object and its current
19325 uint8_t** place = background_mark_stack_tos++;
19327 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
19329 int num_pushed_refs = num_partial_refs;
19330 int num_processed_refs = num_pushed_refs * 16;
19332 go_through_object (method_table(oo), oo, s, ppslot,
19333 start, use_start, (oo + s),
19335 uint8_t* o = *ppslot;
19338 if (background_mark (o,
19339 background_saved_lowest_address,
19340 background_saved_highest_address))
19343 size_t obj_size = size (o);
19344 bpromoted_bytes (thread) += obj_size;
19345 if (contain_pointers_or_collectible (o))
19347 *(background_mark_stack_tos++) = o;
19348 if (--num_pushed_refs == 0)
19351 *place = (uint8_t*)(ppslot+1);
19357 if (--num_processed_refs == 0)
19359 // give foreground GC a chance to run
19360 *place = (uint8_t*)(ppslot + 1);
19366 //we are finished with this object
19374 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
19375 background_min_overflow_address = min (background_min_overflow_address, oo);
19376 background_max_overflow_address = max (background_max_overflow_address, oo);
19380 #ifdef SORT_MARK_STACK
19381 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
19383 rqsort1 (sorted_tos, background_mark_stack_tos-1);
19384 sorted_tos = background_mark_stack_tos-1;
19386 #endif //SORT_MARK_STACK
19388 #ifdef COLLECTIBLE_CLASS
19390 #endif // COLLECTIBLE_CLASS
19393 if (!(background_mark_stack_tos == background_mark_stack_array))
19395 oo = *(--background_mark_stack_tos);
19397 #ifdef SORT_MARK_STACK
19398 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
19399 #endif //SORT_MARK_STACK
19405 assert (background_mark_stack_tos == background_mark_stack_array);
19410 //this version is different than the foreground GC because
19411 //it can't keep pointers to the inside of an object
19412 //while calling background_mark_simple1. The object could be moved
19413 //by an intervening foreground gc.
19414 //this method assumes that *po is in the [low. high[ range
19416 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
19418 #ifdef MULTIPLE_HEAPS
19419 #else //MULTIPLE_HEAPS
19420 const int thread = 0;
19421 #endif //MULTIPLE_HEAPS
19423 dprintf (3, ("bmarking %Ix", o));
19425 if (background_mark1 (o))
19428 size_t s = size (o);
19429 bpromoted_bytes (thread) += s;
19431 if (contain_pointers_or_collectible (o))
19433 background_mark_simple1 (o THREAD_NUMBER_ARG);
19441 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
19443 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
19445 background_mark_simple (o THREAD_NUMBER_ARG);
19451 dprintf (3, ("or-%Ix", o));
19457 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
19459 UNREFERENCED_PARAMETER(sc);
19460 //in order to save space on the array, mark the object,
19461 //knowing that it will be visited later
19462 assert (settings.concurrent);
19464 THREAD_NUMBER_FROM_CONTEXT;
19465 #ifndef MULTIPLE_HEAPS
19466 const int thread = 0;
19467 #endif //!MULTIPLE_HEAPS
19469 uint8_t* o = (uint8_t*)*ppObject;
19474 #ifdef DEBUG_DestroyedHandleValue
19475 // we can race with destroy handle during concurrent scan
19476 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
19478 #endif //DEBUG_DestroyedHandleValue
19482 gc_heap* hp = gc_heap::heap_of (o);
19484 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
19489 if (flags & GC_CALL_INTERIOR)
19491 o = hp->find_object (o);
19496 #ifdef FEATURE_CONSERVATIVE_GC
19497 // For conservative GC, a value on stack may point to middle of a free object.
19498 // In this case, we don't need to promote the pointer.
19499 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
19503 #endif //FEATURE_CONSERVATIVE_GC
19506 ((CObjectHeader*)o)->Validate();
19509 //needs to be called before the marking because it is possible for a foreground
19510 //gc to take place during the mark and move the object
19511 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
19513 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
19516 //used by the ephemeral collection to scan the local background structures
19517 //containing references.
19519 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
19525 pSC->thread_number = hn;
19527 BOOL relocate_p = (fn == &GCHeap::Relocate);
19529 dprintf (3, ("Scanning background mark list"));
19532 size_t mark_list_finger = 0;
19533 while (mark_list_finger < c_mark_list_index)
19535 uint8_t** o = &c_mark_list [mark_list_finger];
19538 // We may not be able to calculate the size during relocate as POPO
19539 // may have written over the object.
19540 size_t s = size (*o);
19541 assert (Align (s) >= Align (min_obj_size));
19542 dprintf(3,("background root %Ix", (size_t)*o));
19544 (*fn) ((Object**)o, pSC, 0);
19545 mark_list_finger++;
19548 //scan the mark stack
19549 dprintf (3, ("Scanning background mark stack"));
19551 uint8_t** finger = background_mark_stack_array;
19552 while (finger < background_mark_stack_tos)
19554 if ((finger + 1) < background_mark_stack_tos)
19556 // We need to check for the partial mark case here.
19557 uint8_t* parent_obj = *(finger + 1);
19558 if ((size_t)parent_obj & 1)
19560 uint8_t* place = *finger;
19561 size_t place_offset = 0;
19562 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
19566 *(finger + 1) = real_parent_obj;
19567 place_offset = place - real_parent_obj;
19568 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
19569 (*fn) ((Object**)(finger + 1), pSC, 0);
19570 real_parent_obj = *(finger + 1);
19571 *finger = real_parent_obj + place_offset;
19572 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
19573 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
19577 uint8_t** temp = &real_parent_obj;
19578 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
19579 (*fn) ((Object**)temp, pSC, 0);
19586 dprintf(3,("background root %Ix", (size_t)*finger));
19587 (*fn) ((Object**)finger, pSC, 0);
19592 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
19594 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
19596 // for now we stop at where gen1 started when we started processing
19597 return background_min_soh_overflow_address;
19601 return heap_segment_allocated (seg);
19605 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
19608 BOOL small_object_p)
19612 if (small_object_p)
19614 if (in_range_for_segment (min_add, seg))
19616 // min_add was the beginning of gen1 when we did the concurrent
19617 // overflow. Now we could be in a situation where min_add is
19618 // actually the same as allocated for that segment (because
19619 // we expanded heap), in which case we can not call
19620 // find first on this address or we will AV.
19621 if (min_add >= heap_segment_allocated (seg))
19627 if (concurrent_p &&
19628 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
19630 return background_min_soh_overflow_address;
19634 o = find_first_object (min_add, heap_segment_mem (seg));
19641 o = max (heap_segment_mem (seg), min_add);
19645 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
19646 uint8_t* min_add, uint8_t* max_add,
19651 current_bgc_state = bgc_overflow_soh;
19654 size_t total_marked_objects = 0;
19656 #ifdef MULTIPLE_HEAPS
19657 int thread = heap_number;
19658 #endif //MULTIPLE_HEAPS
19660 exclusive_sync* loh_alloc_lock = 0;
19662 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19663 #ifdef MULTIPLE_HEAPS
19664 // We don't have each heap scan all heaps concurrently because we are worried about
19665 // multiple threads calling things like find_first_object.
19666 int h_start = (concurrent_p ? heap_number : 0);
19667 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
19668 for (int hi = h_start; hi < h_end; hi++)
19670 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
19676 #endif //MULTIPLE_HEAPS
19677 BOOL small_object_segments = TRUE;
19678 loh_alloc_lock = hp->bgc_alloc_lock;
19680 for (int i = condemned_gen_number; i < total_generation_count; i++)
19682 int align_const = get_alignment_constant (small_object_segments);
19683 generation* gen = hp->generation_of (i);
19684 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19685 PREFIX_ASSUME(seg != NULL);
19689 uint8_t* o = hp->background_first_overflow (min_add, seg, concurrent_p, small_object_segments);
19691 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
19693 dprintf (3, ("considering %Ix", (size_t)o));
19697 if (concurrent_p && !small_object_segments)
19699 loh_alloc_lock->bgc_mark_set (o);
19701 if (((CObjectHeader*)o)->IsFree())
19703 s = unused_array_size (o);
19715 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
19717 total_marked_objects++;
19718 go_through_object_cl (method_table(o), o, s, poo,
19719 uint8_t* oo = *poo;
19720 background_mark_object (oo THREAD_NUMBER_ARG);
19724 if (concurrent_p && !small_object_segments)
19726 loh_alloc_lock->bgc_mark_done ();
19729 o = o + Align (s, align_const);
19737 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
19738 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
19740 if (concurrent_p && (seg == hp->saved_overflow_ephemeral_seg))
19745 seg = heap_segment_next_in_range (seg);
19750 current_bgc_state = bgc_overflow_uoh;
19753 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
19754 fire_overflow_event (min_add, max_add, total_marked_objects, i);
19755 if (small_object_segments)
19757 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
19760 total_marked_objects = 0;
19761 small_object_segments = FALSE;
19766 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
19768 BOOL grow_mark_array_p = TRUE;
19772 assert (!processed_soh_overflow_p);
19774 if ((background_max_overflow_address != 0) &&
19775 (background_min_overflow_address != MAX_PTR))
19777 // We have overflow to process but we know we can't process the ephemeral generations
19778 // now (we actually could process till the current gen1 start but since we are going to
19779 // make overflow per segment, for now I'll just stop at the saved gen1 start.
19780 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
19781 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
19782 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
19787 assert ((saved_overflow_ephemeral_seg == 0) ||
19788 ((background_max_soh_overflow_address != 0) &&
19789 (background_min_soh_overflow_address != MAX_PTR)));
19791 if (!processed_soh_overflow_p)
19793 // if there was no more overflow we just need to process what we didn't process
19794 // on the saved ephemeral segment.
19795 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
19797 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
19798 grow_mark_array_p = FALSE;
19801 background_min_overflow_address = min (background_min_overflow_address,
19802 background_min_soh_overflow_address);
19803 background_max_overflow_address = max (background_max_overflow_address,
19804 background_max_soh_overflow_address);
19805 processed_soh_overflow_p = TRUE;
19809 BOOL overflow_p = FALSE;
19811 if ((! ((background_max_overflow_address == 0)) ||
19812 ! ((background_min_overflow_address == MAX_PTR))))
19816 if (grow_mark_array_p)
19818 // Try to grow the array.
19819 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
19821 if ((new_size * sizeof(mark)) > 100*1024)
19823 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19825 new_size = min(new_max_size, new_size);
19828 if ((background_mark_stack_array_length < new_size) &&
19829 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
19831 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
19833 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19836 delete [] background_mark_stack_array;
19837 background_mark_stack_array = tmp;
19838 background_mark_stack_array_length = new_size;
19839 background_mark_stack_tos = background_mark_stack_array;
19845 grow_mark_array_p = TRUE;
19848 uint8_t* min_add = background_min_overflow_address;
19849 uint8_t* max_add = background_max_overflow_address;
19851 background_max_overflow_address = 0;
19852 background_min_overflow_address = MAX_PTR;
19854 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19864 #endif //BACKGROUND_GC
19867 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19869 #ifndef COLLECTIBLE_CLASS
19870 UNREFERENCED_PARAMETER(mark_class_object_p);
19871 BOOL to_mark_class_object = FALSE;
19872 #else //COLLECTIBLE_CLASS
19873 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19874 #endif //COLLECTIBLE_CLASS
19875 if (contain_pointers (oo) || to_mark_class_object)
19877 dprintf(3,( "Marking through %Ix", (size_t)oo));
19878 size_t s = size (oo);
19880 #ifdef COLLECTIBLE_CLASS
19881 if (to_mark_class_object)
19883 uint8_t* class_obj = get_class_object (oo);
19884 mark_object (class_obj THREAD_NUMBER_ARG);
19886 #endif //COLLECTIBLE_CLASS
19888 if (contain_pointers (oo))
19890 go_through_object_nostart (method_table(oo), oo, s, po,
19892 mark_object (o THREAD_NUMBER_ARG);
19898 size_t gc_heap::get_total_heap_size()
19900 size_t total_heap_size = 0;
19902 #ifdef MULTIPLE_HEAPS
19905 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19907 gc_heap* hp2 = gc_heap::g_heaps [hn];
19908 for (int i = max_generation; i < total_generation_count; i++)
19910 total_heap_size += hp2->generation_sizes (hp2->generation_of (i));
19914 for (int i = max_generation; i < total_generation_count; i++)
19916 total_heap_size += generation_sizes (generation_of (i));
19918 #endif //MULTIPLE_HEAPS
19920 return total_heap_size;
19923 size_t gc_heap::get_total_fragmentation()
19925 size_t total_fragmentation = 0;
19927 #ifdef MULTIPLE_HEAPS
19928 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19930 gc_heap* hp = gc_heap::g_heaps[hn];
19931 #else //MULTIPLE_HEAPS
19933 gc_heap* hp = pGenGCHeap;
19934 #endif //MULTIPLE_HEAPS
19935 for (int i = 0; i < total_generation_count; i++)
19937 generation* gen = hp->generation_of (i);
19938 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19942 return total_fragmentation;
19945 size_t gc_heap::get_total_gen_fragmentation (int gen_number)
19947 size_t total_fragmentation = 0;
19949 #ifdef MULTIPLE_HEAPS
19950 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19952 gc_heap* hp = gc_heap::g_heaps[hn];
19953 #else //MULTIPLE_HEAPS
19955 gc_heap* hp = pGenGCHeap;
19956 #endif //MULTIPLE_HEAPS
19957 generation* gen = hp->generation_of (gen_number);
19958 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19961 return total_fragmentation;
19964 size_t gc_heap::get_total_gen_estimated_reclaim (int gen_number)
19966 size_t total_estimated_reclaim = 0;
19968 #ifdef MULTIPLE_HEAPS
19969 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19971 gc_heap* hp = gc_heap::g_heaps[hn];
19972 #else //MULTIPLE_HEAPS
19974 gc_heap* hp = pGenGCHeap;
19975 #endif //MULTIPLE_HEAPS
19976 total_estimated_reclaim += hp->estimated_reclaim (gen_number);
19979 return total_estimated_reclaim;
19982 size_t gc_heap::committed_size()
19984 size_t total_committed = 0;
19986 for (int i = max_generation; i < total_generation_count; i++)
19988 generation* gen = generation_of (i);
19989 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19993 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19994 seg = heap_segment_next (seg);
19998 return total_committed;
20001 size_t gc_heap::get_total_committed_size()
20003 size_t total_committed = 0;
20005 #ifdef MULTIPLE_HEAPS
20008 for (hn = 0; hn < gc_heap::n_heaps; hn++)
20010 gc_heap* hp = gc_heap::g_heaps [hn];
20011 total_committed += hp->committed_size();
20014 total_committed = committed_size();
20015 #endif //MULTIPLE_HEAPS
20017 return total_committed;
20020 size_t gc_heap::uoh_committed_size (int gen_number, size_t* allocated)
20022 generation* gen = generation_of (gen_number);
20023 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
20024 size_t total_committed = 0;
20025 size_t total_allocated = 0;
20029 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
20030 total_allocated += heap_segment_allocated (seg) - (uint8_t*)seg;
20031 seg = heap_segment_next (seg);
20034 *allocated = total_allocated;
20035 return total_committed;
20038 void gc_heap::get_memory_info (uint32_t* memory_load,
20039 uint64_t* available_physical,
20040 uint64_t* available_page_file)
20042 GCToOSInterface::GetMemoryStatus(is_restricted_physical_mem ? total_physical_mem : 0, memory_load, available_physical, available_page_file);
20045 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
20047 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
20048 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
20051 //returns TRUE is an overflow happened.
20052 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
20054 size_t last_promoted_bytes = promoted_bytes (heap_number);
20055 BOOL overflow_p = FALSE;
20057 if ((! (max_overflow_address == 0) ||
20058 ! (min_overflow_address == MAX_PTR)))
20061 // Try to grow the array.
20063 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
20065 if ((new_size * sizeof(mark)) > 100*1024)
20067 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
20069 new_size = min(new_max_size, new_size);
20072 if ((mark_stack_array_length < new_size) &&
20073 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
20075 mark* tmp = new (nothrow) mark [new_size];
20078 delete mark_stack_array;
20079 mark_stack_array = tmp;
20080 mark_stack_array_length = new_size;
20084 uint8_t* min_add = min_overflow_address;
20085 uint8_t* max_add = max_overflow_address;
20086 max_overflow_address = 0;
20087 min_overflow_address = MAX_PTR;
20088 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
20092 size_t current_promoted_bytes = promoted_bytes (heap_number);
20094 if (current_promoted_bytes != last_promoted_bytes)
20095 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
20099 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
20100 uint8_t* min_add, uint8_t* max_add)
20102 #ifdef MULTIPLE_HEAPS
20103 int thread = heap_number;
20104 #endif //MULTIPLE_HEAPS
20105 BOOL full_p = (condemned_gen_number == max_generation);
20107 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
20108 #ifdef MULTIPLE_HEAPS
20109 for (int hi = 0; hi < n_heaps; hi++)
20111 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
20117 #endif //MULTIPLE_HEAPS
20118 int gen_limit = full_p ? total_generation_count : condemned_gen_number + 1;
20120 for (int i = condemned_gen_number; i < gen_limit; i++)
20122 generation* gen = hp->generation_of (i);
20123 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
20124 int align_const = get_alignment_constant (i < uoh_start_generation);
20126 PREFIX_ASSUME(seg != NULL);
20130 uint8_t* o = max (heap_segment_mem (seg), min_add);
20131 uint8_t* end = heap_segment_allocated (seg);
20133 while ((o < end) && (o <= max_add))
20135 assert ((min_add <= o) && (max_add >= o));
20136 dprintf (3, ("considering %Ix", (size_t)o));
20139 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
20142 o = o + Align (size (o), align_const);
20145 seg = heap_segment_next_in_range (seg);
20151 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
20152 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
20153 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
20154 // promotion scan multiple times.
20155 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
20156 // also has the effect of processing any mark stack overflow.
20158 #ifdef MULTIPLE_HEAPS
20159 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
20160 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
20161 // implementations based on whether MULTIPLE_HEAPS is defined or not.
20163 // Define some static variables used for synchronization in the method below. These should really be defined
20164 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
20166 // A note about the synchronization used within this method. Communication between the worker threads is
20167 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
20168 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
20169 // protection of a join.
20170 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
20171 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
20172 static VOLATILE(BOOL) s_fScanRequired;
20173 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
20175 // Whenever we call this method there may have been preceding object promotions. So set
20176 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
20177 // based on the how the scanning proceeded).
20178 s_fUnscannedPromotions = TRUE;
20180 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
20181 // the state of this thread's portion of the dependent handle table. That's because promotions on other
20182 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
20183 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
20184 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
20185 // as all the others or they'll get out of step).
20188 // The various worker threads are all currently racing in this code. We need to work out if at least
20189 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
20190 // dependent handle table when both of the following conditions apply:
20191 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
20192 // object happens to correspond to a primary in one of our handles we might potentially have to
20193 // promote the associated secondary).
20194 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
20196 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
20197 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
20198 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
20199 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
20200 // follows below. Note that we can't read this outside of the join since on any iteration apart from
20201 // the first threads will be racing between reading this value and completing their previous
20202 // iteration's table scan.
20204 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
20205 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
20206 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
20207 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
20208 // we're safely joined.
20209 if (GCScan::GcDhUnpromotedHandlesExist(sc))
20210 s_fUnpromotedHandles = TRUE;
20212 // Synchronize all the threads so we can read our state variables safely. The shared variable
20213 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
20214 // a single thread inside the join.
20215 gc_t_join.join(this, gc_join_scan_dependent_handles);
20216 if (gc_t_join.joined())
20218 // We're synchronized so it's safe to read our shared state variables. We update another shared
20219 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
20220 // the loop. We scan if there has been at least one object promotion since last time and at least
20221 // one thread has a dependent handle table with a potential handle promotion possible.
20222 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
20224 // Reset our shared state variables (ready to be set again on this scan or with a good initial
20225 // value for the next call if we're terminating the loop).
20226 s_fUnscannedPromotions = FALSE;
20227 s_fUnpromotedHandles = FALSE;
20229 if (!s_fScanRequired)
20231 // We're terminating the loop. Perform any last operations that require single threaded access.
20232 if (!initial_scan_p)
20234 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
20235 // load balance if some of the heaps have an abnormally large workload.
20236 uint8_t* all_heaps_max = 0;
20237 uint8_t* all_heaps_min = MAX_PTR;
20239 for (i = 0; i < n_heaps; i++)
20241 if (all_heaps_max < g_heaps[i]->max_overflow_address)
20242 all_heaps_max = g_heaps[i]->max_overflow_address;
20243 if (all_heaps_min > g_heaps[i]->min_overflow_address)
20244 all_heaps_min = g_heaps[i]->min_overflow_address;
20246 for (i = 0; i < n_heaps; i++)
20248 g_heaps[i]->max_overflow_address = all_heaps_max;
20249 g_heaps[i]->min_overflow_address = all_heaps_min;
20254 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
20255 gc_t_join.restart();
20258 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
20259 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
20260 // global flag indicating that at least one object promotion may have occurred (the usual comment
20261 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
20262 // exit the method since we unconditionally set this variable on method entry anyway).
20263 if (process_mark_overflow(condemned_gen_number))
20264 s_fUnscannedPromotions = TRUE;
20266 // If we decided that no scan was required we can terminate the loop now.
20267 if (!s_fScanRequired)
20270 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
20271 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
20272 // could miss noting the promotion of some primary objects).
20273 gc_t_join.join(this, gc_join_rescan_dependent_handles);
20274 if (gc_t_join.joined())
20276 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
20277 gc_t_join.restart();
20280 // If the portion of the dependent handle table managed by this worker has handles that could still be
20281 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
20282 // could require a rescan of handles on this or other workers.
20283 if (GCScan::GcDhUnpromotedHandlesExist(sc))
20284 if (GCScan::GcDhReScan(sc))
20285 s_fUnscannedPromotions = TRUE;
20288 #else //MULTIPLE_HEAPS
20289 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
20290 // threads synchronized.
20291 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
20293 UNREFERENCED_PARAMETER(initial_scan_p);
20295 // Whenever we call this method there may have been preceding object promotions. So set
20296 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
20297 // based on the how the scanning proceeded).
20298 bool fUnscannedPromotions = true;
20300 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
20301 // managed to perform a scan without promoting anything new.
20302 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
20304 // On each iteration of the loop start with the assumption that no further objects have been promoted.
20305 fUnscannedPromotions = false;
20307 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
20308 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
20309 // objects now appear to be promoted and we should set the flag.
20310 if (process_mark_overflow(condemned_gen_number))
20311 fUnscannedPromotions = true;
20313 // Perform the scan and set the flag if any promotions resulted.
20314 if (GCScan::GcDhReScan(sc))
20315 fUnscannedPromotions = true;
20318 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
20319 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
20321 process_mark_overflow(condemned_gen_number);
20323 #endif //MULTIPLE_HEAPS
20325 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
20327 assert (settings.concurrent == FALSE);
20330 sc.thread_number = heap_number;
20331 sc.promotion = TRUE;
20332 sc.concurrent = FALSE;
20334 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
20335 BOOL full_p = (condemned_gen_number == max_generation);
20337 int gen_to_init = condemned_gen_number;
20338 if (condemned_gen_number == max_generation)
20340 gen_to_init = total_generation_count - 1;
20343 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
20345 dynamic_data* dd = dynamic_data_of (gen_idx);
20346 dd_begin_data_size (dd) = generation_size (gen_idx) -
20347 dd_fragmentation (dd) -
20348 Align (size (generation_allocation_start (generation_of (gen_idx))));
20349 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
20350 dd_survived_size (dd) = 0;
20351 dd_pinned_survived_size (dd) = 0;
20352 dd_artificial_pinned_survived_size (dd) = 0;
20353 dd_added_pinned_size (dd) = 0;
20355 dd_padding_size (dd) = 0;
20356 #endif //SHORT_PLUGS
20357 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
20358 dd_num_npinned_plugs (dd) = 0;
20359 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
20362 if (gen0_must_clear_bricks > 0)
20363 gen0_must_clear_bricks--;
20365 size_t last_promoted_bytes = 0;
20367 promoted_bytes (heap_number) = 0;
20368 reset_mark_stack();
20371 memset (&snoop_stat, 0, sizeof(snoop_stat));
20372 snoop_stat.heap_index = heap_number;
20373 #endif //SNOOP_STATS
20378 //initialize the mark stack
20379 for (int i = 0; i < max_snoop_level; i++)
20381 ((uint8_t**)(mark_stack_array))[i] = 0;
20384 mark_stack_busy() = 1;
20386 #endif //MH_SC_MARK
20388 static uint32_t num_sizedrefs = 0;
20391 static BOOL do_mark_steal_p = FALSE;
20392 #endif //MH_SC_MARK
20394 #ifdef FEATURE_CARD_MARKING_STEALING
20395 reset_card_marking_enumerators();
20396 #endif // FEATURE_CARD_MARKING_STEALING
20398 #ifdef MULTIPLE_HEAPS
20399 gc_t_join.join(this, gc_join_begin_mark_phase);
20400 if (gc_t_join.joined())
20401 #endif //MULTIPLE_HEAPS
20403 maxgen_size_inc_p = false;
20405 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
20407 #ifdef MULTIPLE_HEAPS
20412 size_t total_heap_size = get_total_heap_size();
20414 if (total_heap_size > (100 * 1024 * 1024))
20416 do_mark_steal_p = TRUE;
20420 do_mark_steal_p = FALSE;
20425 do_mark_steal_p = FALSE;
20427 #endif //MH_SC_MARK
20429 gc_t_join.restart();
20430 #endif //MULTIPLE_HEAPS
20435 //set up the mark lists from g_mark_list
20436 assert (g_mark_list);
20437 #ifdef MULTIPLE_HEAPS
20438 mark_list = &g_mark_list [heap_number*mark_list_size];
20440 mark_list = g_mark_list;
20441 #endif //MULTIPLE_HEAPS
20442 //dont use the mark list for full gc
20443 //because multiple segments are more complex to handle and the list
20444 //is likely to overflow
20445 if (condemned_gen_number < max_generation)
20446 mark_list_end = &mark_list [mark_list_size-1];
20448 mark_list_end = &mark_list [0];
20449 mark_list_index = &mark_list [0];
20452 #ifndef MULTIPLE_HEAPS
20453 shigh = (uint8_t*) 0;
20455 #endif //MULTIPLE_HEAPS
20457 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
20459 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20460 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
20461 last_promoted_bytes = promoted_bytes (heap_number);
20463 #ifdef MULTIPLE_HEAPS
20464 gc_t_join.join(this, gc_join_scan_sizedref_done);
20465 if (gc_t_join.joined())
20467 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
20468 gc_t_join.restart();
20470 #endif //MULTIPLE_HEAPS
20473 dprintf(3,("Marking Roots"));
20475 GCScan::GcScanRoots(GCHeap::Promote,
20476 condemned_gen_number, max_generation,
20479 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
20480 last_promoted_bytes = promoted_bytes (heap_number);
20482 #ifdef BACKGROUND_GC
20483 if (gc_heap::background_running_p())
20485 scan_background_roots (GCHeap::Promote, heap_number, &sc);
20487 #endif //BACKGROUND_GC
20489 #ifdef FEATURE_PREMORTEM_FINALIZATION
20490 dprintf(3, ("Marking finalization data"));
20491 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
20492 #endif // FEATURE_PREMORTEM_FINALIZATION
20494 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
20495 last_promoted_bytes = promoted_bytes (heap_number);
20500 dprintf(3,("Marking handle table"));
20501 GCScan::GcScanHandles(GCHeap::Promote,
20502 condemned_gen_number, max_generation,
20504 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
20505 last_promoted_bytes = promoted_bytes (heap_number);
20509 size_t promoted_before_cards = promoted_bytes (heap_number);
20510 dprintf (3, ("before cards: %Id", promoted_before_cards));
20516 #ifdef MULTIPLE_HEAPS
20517 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
20519 #endif //MULTIPLE_HEAPS
20521 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
20522 // If we are manually managing card bundles, every write to the card table should already be
20523 // accounted for in the card bundle table so there's nothing to update here.
20524 update_card_table_bundle();
20526 if (card_bundles_enabled())
20528 verify_card_bundles();
20531 #ifdef MULTIPLE_HEAPS
20532 gc_t_join.r_restart();
20534 #endif //MULTIPLE_HEAPS
20535 #endif //CARD_BUNDLE
20537 card_fn mark_object_fn = &gc_heap::mark_object_simple;
20538 #ifdef HEAP_ANALYZE
20539 heap_analyze_success = TRUE;
20540 if (heap_analyze_enabled)
20542 internal_root_array_index = 0;
20544 current_obj_size = 0;
20545 mark_object_fn = &gc_heap::ha_mark_object_simple;
20547 #endif //HEAP_ANALYZE
20549 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
20550 if (!card_mark_done_soh)
20551 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
20553 dprintf (3, ("Marking cross generation pointers on heap %d", heap_number));
20554 mark_through_cards_for_segments(mark_object_fn, FALSE THIS_ARG);
20555 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
20556 card_mark_done_soh = true;
20557 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
20560 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
20561 if (!card_mark_done_uoh)
20562 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
20564 dprintf (3, ("Marking cross generation pointers for uoh objects on heap %d", heap_number));
20565 for (int i = uoh_start_generation; i < total_generation_count; i++)
20567 #ifndef ALLOW_REFERENCES_IN_POH
20568 if (i != poh_generation)
20569 #endif //ALLOW_REFERENCES_IN_POH
20570 mark_through_cards_for_uoh_objects(mark_object_fn, i, FALSE THIS_ARG);
20573 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
20574 card_mark_done_uoh = true;
20575 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
20578 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
20579 // check the other heaps cyclically and try to help out where the marking isn't done
20580 for (int i = 0; i < gc_heap::n_heaps; i++)
20582 int heap_number_to_look_at = (i + heap_number) % gc_heap::n_heaps;
20583 gc_heap* hp = gc_heap::g_heaps[heap_number_to_look_at];
20584 if (!hp->card_mark_done_soh)
20586 dprintf(3, ("Marking cross generation pointers on heap %d", hp->heap_number));
20587 hp->mark_through_cards_for_segments(mark_object_fn, FALSE THIS_ARG);
20588 hp->card_mark_done_soh = true;
20591 if (!hp->card_mark_done_uoh)
20593 dprintf(3, ("Marking cross generation pointers for large objects on heap %d", hp->heap_number));
20594 for (int i = uoh_start_generation; i < total_generation_count; i++)
20596 #ifndef ALLOW_REFERENCES_IN_POH
20597 if (i != poh_generation)
20598 #endif //ALLOW_REFERENCES_IN_POH
20599 hp->mark_through_cards_for_uoh_objects(mark_object_fn, i, FALSE THIS_ARG);
20602 hp->card_mark_done_uoh = true;
20605 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
20607 dprintf (3, ("marked by cards: %Id",
20608 (promoted_bytes (heap_number) - promoted_before_cards)));
20609 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
20610 last_promoted_bytes = promoted_bytes (heap_number);
20615 if (do_mark_steal_p)
20619 #endif //MH_SC_MARK
20621 // Dependent handles need to be scanned with a special algorithm (see the header comment on
20622 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
20623 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
20624 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
20625 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
20626 // iterations if required and will also perform processing of any mark stack overflow once the dependent
20627 // handle table has been fully promoted.
20628 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20629 scan_dependent_handles(condemned_gen_number, &sc, true);
20631 #ifdef MULTIPLE_HEAPS
20632 dprintf(3, ("Joining for short weak handle scan"));
20633 gc_t_join.join(this, gc_join_null_dead_short_weak);
20634 if (gc_t_join.joined())
20635 #endif //MULTIPLE_HEAPS
20637 #ifdef HEAP_ANALYZE
20638 heap_analyze_enabled = FALSE;
20639 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
20640 #endif // HEAP_ANALYZE
20641 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
20643 #ifdef MULTIPLE_HEAPS
20646 // we used r_join and need to reinitialize states for it here.
20647 gc_t_join.r_init();
20650 dprintf(3, ("Starting all gc thread for short weak handle scan"));
20651 gc_t_join.restart();
20652 #endif //MULTIPLE_HEAPS
20655 #ifdef FEATURE_CARD_MARKING_STEALING
20656 reset_card_marking_enumerators();
20657 #endif // FEATURE_CARD_MARKING_STEALING
20659 // null out the target of short weakref that were not promoted.
20660 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
20662 // MTHTS: keep by single thread
20663 #ifdef MULTIPLE_HEAPS
20664 dprintf(3, ("Joining for finalization"));
20665 gc_t_join.join(this, gc_join_scan_finalization);
20666 if (gc_t_join.joined())
20668 dprintf(3, ("Starting all gc thread for Finalization"));
20669 gc_t_join.restart();
20671 #endif //MULTIPLE_HEAPS
20673 //Handle finalization.
20674 size_t promoted_bytes_live = promoted_bytes (heap_number);
20676 #ifdef FEATURE_PREMORTEM_FINALIZATION
20677 dprintf (3, ("Finalize marking"));
20678 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
20680 GCToEEInterface::DiagWalkFReachableObjects(__this);
20681 #endif // FEATURE_PREMORTEM_FINALIZATION
20683 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
20684 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
20685 scan_dependent_handles(condemned_gen_number, &sc, false);
20687 #ifdef MULTIPLE_HEAPS
20688 dprintf(3, ("Joining for weak pointer deletion"));
20689 gc_t_join.join(this, gc_join_null_dead_long_weak);
20690 if (gc_t_join.joined())
20692 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
20693 gc_t_join.restart();
20695 #endif //MULTIPLE_HEAPS
20697 // null out the target of long weakref that were not promoted.
20698 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20700 // MTHTS: keep by single thread
20701 #ifdef MULTIPLE_HEAPS
20703 #ifdef PARALLEL_MARK_LIST_SORT
20704 // unsigned long start = GetCycleCount32();
20706 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
20707 #endif //PARALLEL_MARK_LIST_SORT
20710 dprintf (3, ("Joining for sync block cache entry scanning"));
20711 gc_t_join.join(this, gc_join_null_dead_syncblk);
20712 if (gc_t_join.joined())
20713 #endif //MULTIPLE_HEAPS
20715 // scan for deleted entries in the syncblk cache
20716 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
20718 #ifdef MULTIPLE_HEAPS
20719 #if defined(MARK_LIST) && !defined(PARALLEL_MARK_LIST_SORT)
20720 //compact g_mark_list and sort it.
20721 combine_mark_lists();
20722 #endif //MARK_LIST && !PARALLEL_MARK_LIST_SORT
20724 //decide on promotion
20725 if (!settings.promotion)
20728 for (int n = 0; n <= condemned_gen_number;n++)
20730 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
20733 for (int i = 0; i < n_heaps; i++)
20735 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
20737 size_t older_gen_size = (dd_current_size (dd) +
20738 (dd_desired_allocation (dd) -
20739 dd_new_allocation (dd)));
20741 if ((m > (older_gen_size)) ||
20742 (promoted_bytes (i) > m))
20744 settings.promotion = TRUE;
20750 if (do_mark_steal_p)
20752 size_t objects_checked_count = 0;
20753 size_t zero_ref_count = 0;
20754 size_t objects_marked_count = 0;
20755 size_t check_level_count = 0;
20756 size_t busy_count = 0;
20757 size_t interlocked_count = 0;
20758 size_t partial_mark_parent_count = 0;
20759 size_t stolen_or_pm_count = 0;
20760 size_t stolen_entry_count = 0;
20761 size_t pm_not_ready_count = 0;
20762 size_t normal_count = 0;
20763 size_t stack_bottom_clear_count = 0;
20765 for (int i = 0; i < n_heaps; i++)
20767 gc_heap* hp = g_heaps[i];
20768 hp->print_snoop_stat();
20769 objects_checked_count += hp->snoop_stat.objects_checked_count;
20770 zero_ref_count += hp->snoop_stat.zero_ref_count;
20771 objects_marked_count += hp->snoop_stat.objects_marked_count;
20772 check_level_count += hp->snoop_stat.check_level_count;
20773 busy_count += hp->snoop_stat.busy_count;
20774 interlocked_count += hp->snoop_stat.interlocked_count;
20775 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
20776 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
20777 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
20778 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
20779 normal_count += hp->snoop_stat.normal_count;
20780 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
20785 printf ("-------total stats-------\n");
20786 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
20787 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
20788 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
20789 objects_checked_count,
20791 objects_marked_count,
20795 partial_mark_parent_count,
20796 stolen_or_pm_count,
20797 stolen_entry_count,
20798 pm_not_ready_count,
20800 stack_bottom_clear_count);
20802 #endif //SNOOP_STATS
20804 dprintf(3, ("Starting all threads for end of mark phase"));
20805 gc_t_join.restart();
20806 #else //MULTIPLE_HEAPS
20808 //decide on promotion
20809 if (!settings.promotion)
20812 for (int n = 0; n <= condemned_gen_number;n++)
20814 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
20816 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
20818 size_t older_gen_size = (dd_current_size (dd) +
20819 (dd_desired_allocation (dd) -
20820 dd_new_allocation (dd)));
20822 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
20823 m, promoted_bytes (heap_number), older_gen_size));
20825 if ((m > older_gen_size) ||
20826 (promoted_bytes (heap_number) > m))
20828 settings.promotion = TRUE;
20831 #endif //MULTIPLE_HEAPS
20834 #ifdef MULTIPLE_HEAPS
20836 #ifdef PARALLEL_MARK_LIST_SORT
20837 // start = GetCycleCount32();
20838 merge_mark_lists();
20839 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
20840 #endif //PARALLEL_MARK_LIST_SORT
20842 #endif //MULTIPLE_HEAPS
20844 #ifdef BACKGROUND_GC
20845 total_promoted_bytes = promoted_bytes (heap_number);
20846 #endif //BACKGROUND_GC
20848 promoted_bytes (heap_number) -= promoted_bytes_live;
20850 dprintf(2,("---- End of mark phase ----"));
20854 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject)
20856 dprintf (3, ("Pinning %Ix->%Ix", (size_t)ppObject, (size_t)o));
20859 #ifdef FEATURE_EVENT_TRACE
20860 if(EVENT_ENABLED(PinObjectAtGCTime))
20862 fire_etw_pin_object_event(o, ppObject);
20864 #endif // FEATURE_EVENT_TRACE
20866 num_pinned_objects++;
20869 size_t gc_heap::get_total_pinned_objects()
20871 #ifdef MULTIPLE_HEAPS
20872 size_t total_num_pinned_objects = 0;
20873 for (int i = 0; i < gc_heap::n_heaps; i++)
20875 gc_heap* hp = gc_heap::g_heaps[i];
20876 total_num_pinned_objects += hp->num_pinned_objects;
20878 return total_num_pinned_objects;
20879 #else //MULTIPLE_HEAPS
20880 return num_pinned_objects;
20881 #endif //MULTIPLE_HEAPS
20884 void gc_heap::reset_mark_stack ()
20886 reset_pinned_queue();
20887 max_overflow_address = 0;
20888 min_overflow_address = MAX_PTR;
20891 #ifdef FEATURE_STRUCTALIGN
20893 // The word with left child, right child, and align info is laid out as follows:
20895 // | upper short word | lower short word |
20896 // |<------------> <----->|<------------> <----->|
20897 // | left child info hi| right child info lo|
20898 // x86: | 10 bits 6 bits| 10 bits 6 bits|
20900 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
20902 // The "align info" encodes two numbers: the required alignment (a power of two)
20903 // and the misalignment (the number of machine words the destination address needs
20904 // to be adjusted by to provide alignment - so this number is always smaller than
20905 // the required alignment). Thus, the two can be represented as the "logical or"
20906 // of the two numbers. Note that the actual pad is computed from the misalignment
20907 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20910 // The number of bits in a brick.
20911 #if defined (TARGET_AMD64)
20912 #define brick_bits (12)
20914 #define brick_bits (11)
20915 #endif //TARGET_AMD64
20916 C_ASSERT(brick_size == (1 << brick_bits));
20918 // The number of bits needed to represent the offset to a child node.
20919 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20920 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20922 // The number of bits in each of the pad hi, pad lo fields.
20923 #define pad_bits (sizeof(short) * 8 - child_bits)
20925 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20926 #define pad_mask ((1 << pad_bits) - 1)
20927 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20928 #else // FEATURE_STRUCTALIGN
20929 #define child_from_short(w) (w)
20930 #endif // FEATURE_STRUCTALIGN
20933 short node_left_child(uint8_t* node)
20935 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20939 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20941 assert (val > -(ptrdiff_t)brick_size);
20942 assert (val < (ptrdiff_t)brick_size);
20943 assert (Aligned (val));
20944 #ifdef FEATURE_STRUCTALIGN
20945 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20946 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20947 #else // FEATURE_STRUCTALIGN
20948 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20949 #endif // FEATURE_STRUCTALIGN
20950 assert (node_left_child (node) == val);
20954 short node_right_child(uint8_t* node)
20956 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20960 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20962 assert (val > -(ptrdiff_t)brick_size);
20963 assert (val < (ptrdiff_t)brick_size);
20964 assert (Aligned (val));
20965 #ifdef FEATURE_STRUCTALIGN
20966 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20967 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20968 #else // FEATURE_STRUCTALIGN
20969 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20970 #endif // FEATURE_STRUCTALIGN
20971 assert (node_right_child (node) == val);
20974 #ifdef FEATURE_STRUCTALIGN
20975 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20977 // Extract the single-number aligninfo from the fields.
20978 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20979 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20980 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20981 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20983 // Replicate the topmost bit into all lower bits.
20984 ptrdiff_t x = aligninfo;
20990 // Clear all bits but the highest.
20991 requiredAlignment = (int)(x ^ (x >> 1));
20992 pad = aligninfo - requiredAlignment;
20993 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20997 ptrdiff_t node_alignpad (uint8_t* node)
20999 int requiredAlignment;
21000 ptrdiff_t alignpad;
21001 node_aligninfo (node, requiredAlignment, alignpad);
21005 void clear_node_aligninfo (uint8_t* node)
21007 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
21008 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
21011 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
21013 // Encode the alignment requirement and alignment offset as a single number
21014 // as described above.
21015 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
21016 assert (Aligned (aligninfo));
21017 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
21018 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
21020 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
21021 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
21022 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
21024 ptrdiff_t lo = aligninfo_shifted & pad_mask;
21025 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
21026 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
21029 int requiredAlignment2;
21031 node_aligninfo (node, requiredAlignment2, pad2);
21032 assert (requiredAlignment == requiredAlignment2);
21033 assert (pad == pad2);
21036 #endif // FEATURE_STRUCTALIGN
21039 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
21041 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
21046 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
21048 return (((loh_obj_and_pad*)node)[-1].reloc);
21052 ptrdiff_t node_relocation_distance (uint8_t* node)
21054 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
21058 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
21060 assert (val == (val & ~3));
21061 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
21062 //clear the left bit and the relocation field
21067 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
21069 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
21071 #ifndef FEATURE_STRUCTALIGN
21072 void set_node_realigned(uint8_t* node)
21074 ((plug_and_reloc*)(node))[-1].reloc |= 1;
21077 void clear_node_realigned(uint8_t* node)
21079 #ifdef RESPECT_LARGE_ALIGNMENT
21080 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
21082 UNREFERENCED_PARAMETER(node);
21083 #endif //RESPECT_LARGE_ALIGNMENT
21085 #endif // FEATURE_STRUCTALIGN
21088 size_t node_gap_size (uint8_t* node)
21090 return ((plug_and_gap *)node)[-1].gap;
21093 void set_gap_size (uint8_t* node, size_t size)
21095 assert (Aligned (size));
21097 // clear the 2 uint32_t used by the node.
21098 ((plug_and_gap *)node)[-1].reloc = 0;
21099 ((plug_and_gap *)node)[-1].lr =0;
21100 ((plug_and_gap *)node)[-1].gap = size;
21102 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
21106 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
21107 uint8_t* tree, uint8_t* last_node)
21109 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
21110 (size_t)new_node, brick_of(new_node),
21111 (size_t)tree, brick_of(tree),
21112 (size_t)last_node, brick_of(last_node),
21114 if (power_of_two_p (sequence_number))
21116 set_node_left_child (new_node, (tree - new_node));
21117 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
21122 if (oddp (sequence_number))
21124 set_node_right_child (last_node, (new_node - last_node));
21125 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
21129 uint8_t* earlier_node = tree;
21130 size_t imax = logcount(sequence_number) - 2;
21131 for (size_t i = 0; i != imax; i++)
21133 earlier_node = earlier_node + node_right_child (earlier_node);
21135 int tmp_offset = node_right_child (earlier_node);
21136 assert (tmp_offset); // should never be empty
21137 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
21138 set_node_right_child (earlier_node, (new_node - earlier_node));
21140 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
21141 new_node, ((earlier_node + tmp_offset ) - new_node),
21142 earlier_node, (new_node - earlier_node)));
21148 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
21149 uint8_t* x, uint8_t* plug_end)
21151 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
21152 tree, current_brick, x, plug_end));
21156 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
21157 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
21158 set_brick (current_brick, (tree - brick_address (current_brick)));
21162 dprintf (3, ("b- %Ix->-1", current_brick));
21163 set_brick (current_brick, -1);
21165 size_t b = 1 + current_brick;
21166 ptrdiff_t offset = 0;
21167 size_t last_br = brick_of (plug_end-1);
21168 current_brick = brick_of (x-1);
21169 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
21170 while (b <= current_brick)
21174 set_brick (b, --offset);
21182 return brick_of (x);
21185 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
21188 // We should never demote big plugs to gen0.
21189 if (gen == youngest_generation)
21191 heap_segment* seg = ephemeral_heap_segment;
21192 size_t mark_stack_large_bos = mark_stack_bos;
21193 size_t large_plug_pos = 0;
21194 while (mark_stack_large_bos < mark_stack_tos)
21196 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
21198 while (mark_stack_bos <= mark_stack_large_bos)
21200 size_t entry = deque_pinned_plug();
21201 size_t len = pinned_len (pinned_plug_of (entry));
21202 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
21203 if (len > demotion_plug_len_th)
21205 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
21207 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
21208 assert(mark_stack_array[entry].len == 0 ||
21209 mark_stack_array[entry].len >= Align(min_obj_size));
21210 generation_allocation_pointer (consing_gen) = plug + len;
21211 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
21212 set_allocator_next_pin (consing_gen);
21216 mark_stack_large_bos++;
21219 #endif // HOST_64BIT
21221 generation_plan_allocation_start (gen) =
21222 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
21223 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
21224 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
21225 if (next_plug_to_allocate)
21227 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
21228 if (allocation_left > dist_to_next_plug)
21230 allocation_left = dist_to_next_plug;
21233 if (allocation_left < Align (min_obj_size))
21235 generation_plan_allocation_start_size (gen) += allocation_left;
21236 generation_allocation_pointer (consing_gen) += allocation_left;
21239 dprintf (2, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
21240 generation_plan_allocation_start (gen),
21241 generation_plan_allocation_start_size (gen),
21242 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
21243 next_plug_to_allocate));
21246 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
21248 BOOL adjacentp = FALSE;
21250 generation_plan_allocation_start (gen) =
21251 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
21254 #endif //SHORT_PLUGS
21255 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
21257 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
21258 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
21259 if ((allocation_left < Align (min_obj_size)) &&
21260 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
21262 generation_plan_allocation_start_size (gen) += allocation_left;
21263 generation_allocation_pointer (consing_gen) += allocation_left;
21266 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
21267 generation_plan_allocation_start (consing_gen),
21268 generation_allocation_pointer (consing_gen),
21269 generation_allocation_limit (consing_gen)));
21272 void gc_heap::plan_generation_starts (generation*& consing_gen)
21274 //make sure that every generation has a planned allocation start
21275 int gen_number = settings.condemned_generation;
21276 while (gen_number >= 0)
21278 if (gen_number < max_generation)
21280 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
21282 generation* gen = generation_of (gen_number);
21283 if (0 == generation_plan_allocation_start (gen))
21285 plan_generation_start (gen, consing_gen, 0);
21286 assert (generation_plan_allocation_start (gen));
21290 // now we know the planned allocation size
21291 heap_segment_plan_allocated (ephemeral_heap_segment) =
21292 generation_allocation_pointer (consing_gen);
21295 void gc_heap::advance_pins_for_demotion (generation* gen)
21297 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
21298 heap_segment* seg = ephemeral_heap_segment;
21300 if ((!(pinned_plug_que_empty_p())))
21302 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
21303 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
21304 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
21305 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
21306 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
21307 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
21309 while (!pinned_plug_que_empty_p() &&
21310 (pinned_plug (oldest_pin()) < original_youngest_start))
21312 size_t entry = deque_pinned_plug();
21313 size_t len = pinned_len (pinned_plug_of (entry));
21314 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
21315 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
21316 assert(mark_stack_array[entry].len == 0 ||
21317 mark_stack_array[entry].len >= Align(min_obj_size));
21318 generation_allocation_pointer (gen) = plug + len;
21319 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21320 set_allocator_next_pin (gen);
21322 //Add the size of the pinned plug to the right pinned allocations
21323 //find out which gen this pinned plug came from
21324 int frgn = object_gennum (plug);
21325 if ((frgn != (int)max_generation) && settings.promotion)
21327 int togn = object_gennum_plan (plug);
21328 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
21331 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
21335 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
21336 pinned_len (pinned_plug_of (entry)), plug, len));
21339 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
21340 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
21344 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
21345 int& active_new_gen_number,
21346 int& active_old_gen_number,
21347 generation*& consing_gen,
21348 BOOL& allocate_in_condemned)
21351 if ((active_old_gen_number > 0) &&
21352 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
21354 dprintf (2, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
21356 if (!pinned_plug_que_empty_p())
21358 dprintf (2, ("oldest pin: %Ix(%Id)",
21359 pinned_plug (oldest_pin()),
21360 (x - pinned_plug (oldest_pin()))));
21363 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
21365 active_new_gen_number--;
21368 active_old_gen_number--;
21369 assert ((!settings.promotion) || (active_new_gen_number>0));
21371 if (active_new_gen_number == (max_generation - 1))
21373 #ifdef FREE_USAGE_STATS
21374 if (settings.condemned_generation == max_generation)
21376 // We need to do this before we skip the rest of the pinned plugs.
21377 generation* gen_2 = generation_of (max_generation);
21378 generation* gen_1 = generation_of (max_generation - 1);
21380 size_t total_num_pinned_free_spaces_left = 0;
21382 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
21383 for (int j = 0; j < NUM_GEN_POWER2; j++)
21385 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
21389 gen_2->gen_current_pinned_free_spaces[j],
21390 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
21391 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
21393 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
21396 float pinned_free_list_efficiency = 0;
21397 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
21398 if (total_pinned_free_space != 0)
21400 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
21403 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
21405 generation_allocated_in_pinned_free (gen_2),
21406 total_pinned_free_space,
21407 (int)(pinned_free_list_efficiency * 100),
21408 generation_pinned_free_obj_space (gen_2),
21409 total_num_pinned_free_spaces_left));
21411 #endif //FREE_USAGE_STATS
21413 //Go past all of the pinned plugs for this generation.
21414 while (!pinned_plug_que_empty_p() &&
21415 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
21417 size_t entry = deque_pinned_plug();
21418 mark* m = pinned_plug_of (entry);
21419 uint8_t* plug = pinned_plug (m);
21420 size_t len = pinned_len (m);
21421 // detect pinned block in different segment (later) than
21422 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
21423 // adjust the allocation segment along the way (at the end it will
21424 // be the ephemeral segment.
21425 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
21427 PREFIX_ASSUME(nseg != NULL);
21429 while (!((plug >= generation_allocation_pointer (consing_gen))&&
21430 (plug < heap_segment_allocated (nseg))))
21432 //adjust the end of the segment to be the end of the plug
21433 assert (generation_allocation_pointer (consing_gen)>=
21434 heap_segment_mem (nseg));
21435 assert (generation_allocation_pointer (consing_gen)<=
21436 heap_segment_committed (nseg));
21438 heap_segment_plan_allocated (nseg) =
21439 generation_allocation_pointer (consing_gen);
21440 //switch allocation segment
21441 nseg = heap_segment_next_rw (nseg);
21442 generation_allocation_segment (consing_gen) = nseg;
21443 //reset the allocation pointer and limits
21444 generation_allocation_pointer (consing_gen) =
21445 heap_segment_mem (nseg);
21447 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
21448 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
21449 generation_allocation_pointer (consing_gen) = plug + len;
21450 generation_allocation_limit (consing_gen) =
21451 generation_allocation_pointer (consing_gen);
21453 allocate_in_condemned = TRUE;
21454 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
21457 if (active_new_gen_number != max_generation)
21459 if (active_new_gen_number == (max_generation - 1))
21461 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
21462 if (!demote_gen1_p)
21463 advance_pins_for_demotion (consing_gen);
21466 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
21468 dprintf (2, ("process eph: allocated gen%d start at %Ix",
21469 active_new_gen_number,
21470 generation_plan_allocation_start (generation_of (active_new_gen_number))));
21472 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
21474 uint8_t* pplug = pinned_plug (oldest_pin());
21475 if (object_gennum (pplug) > 0)
21477 demotion_low = pplug;
21478 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
21482 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
21490 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
21492 uint8_t* o = heap_segment_mem (seg);
21493 while (o < heap_segment_allocated (seg))
21499 o = o + Align (size (o));
21503 #ifdef FEATURE_BASICFREEZE
21504 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
21506 //go through all of the segment in range and reset the mark bit
21507 heap_segment* seg = start_seg;
21511 if (heap_segment_read_only_p (seg) &&
21512 heap_segment_in_range_p (seg))
21514 #ifdef BACKGROUND_GC
21515 if (settings.concurrent)
21517 seg_clear_mark_array_bits_soh (seg);
21521 seg_clear_mark_bits (seg);
21523 #else //BACKGROUND_GC
21524 seg_clear_mark_bits (seg);
21525 #endif //BACKGROUND_GC
21527 seg = heap_segment_next (seg);
21530 #endif // FEATURE_BASICFREEZE
21532 #ifdef FEATURE_LOH_COMPACTION
21534 BOOL gc_heap::loh_pinned_plug_que_empty_p()
21536 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
21539 void gc_heap::loh_set_allocator_next_pin()
21541 if (!(loh_pinned_plug_que_empty_p()))
21543 mark* oldest_entry = loh_oldest_pin();
21544 uint8_t* plug = pinned_plug (oldest_entry);
21545 generation* gen = large_object_generation;
21546 if ((plug >= generation_allocation_pointer (gen)) &&
21547 (plug < generation_allocation_limit (gen)))
21549 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
21552 assert (!((plug < generation_allocation_pointer (gen)) &&
21553 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
21557 size_t gc_heap::loh_deque_pinned_plug ()
21559 size_t m = loh_pinned_queue_bos;
21560 loh_pinned_queue_bos++;
21565 mark* gc_heap::loh_pinned_plug_of (size_t bos)
21567 return &loh_pinned_queue[bos];
21571 mark* gc_heap::loh_oldest_pin()
21573 return loh_pinned_plug_of (loh_pinned_queue_bos);
21576 // If we can't grow the queue, then don't compact.
21577 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
21579 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
21581 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
21583 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
21588 dprintf (3, (" P: %Ix(%Id)", plug, len));
21589 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
21592 loh_pinned_queue_tos++;
21593 loh_set_allocator_next_pin();
21598 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
21600 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
21602 (2* AlignQword (loh_padding_obj_size) + size),
21605 (alloc_limit - alloc_pointer)));
21607 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
21610 uint8_t* gc_heap::loh_allocate_in_condemned (size_t size)
21612 generation* gen = large_object_generation;
21613 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
21614 generation_allocation_pointer (gen),
21615 generation_allocation_limit (gen),
21620 heap_segment* seg = generation_allocation_segment (gen);
21621 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
21623 if ((!(loh_pinned_plug_que_empty_p()) &&
21624 (generation_allocation_limit (gen) ==
21625 pinned_plug (loh_oldest_pin()))))
21627 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21628 size_t len = pinned_len (m);
21629 uint8_t* plug = pinned_plug (m);
21630 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21631 pinned_len (m) = plug - generation_allocation_pointer (gen);
21632 generation_allocation_pointer (gen) = plug + len;
21634 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21635 loh_set_allocator_next_pin();
21636 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
21637 generation_allocation_pointer (gen),
21638 generation_allocation_limit (gen),
21639 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21644 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
21646 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21647 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
21651 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
21653 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21654 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21655 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
21659 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
21660 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
21662 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
21663 (generation_allocation_pointer (gen) + size)));
21665 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21666 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21668 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
21669 generation_allocation_pointer (gen),
21670 generation_allocation_limit (gen),
21671 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21675 heap_segment* next_seg = heap_segment_next (seg);
21676 assert (generation_allocation_pointer (gen)>=
21677 heap_segment_mem (seg));
21678 // Verify that all pinned plugs for this segment are consumed
21679 if (!loh_pinned_plug_que_empty_p() &&
21680 ((pinned_plug (loh_oldest_pin()) <
21681 heap_segment_allocated (seg)) &&
21682 (pinned_plug (loh_oldest_pin()) >=
21683 generation_allocation_pointer (gen))))
21685 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
21686 pinned_plug (loh_oldest_pin())));
21687 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
21690 assert (generation_allocation_pointer (gen)>=
21691 heap_segment_mem (seg));
21692 assert (generation_allocation_pointer (gen)<=
21693 heap_segment_committed (seg));
21694 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
21698 // for LOH do we want to try starting from the first LOH every time though?
21699 generation_allocation_segment (gen) = next_seg;
21700 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
21701 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21703 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
21704 generation_allocation_pointer (gen),
21705 generation_allocation_limit (gen),
21706 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21710 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
21716 loh_set_allocator_next_pin();
21718 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
21719 generation_allocation_pointer (gen),
21720 generation_allocation_limit (gen),
21721 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21728 assert (generation_allocation_pointer (gen)>=
21729 heap_segment_mem (generation_allocation_segment (gen)));
21730 uint8_t* result = generation_allocation_pointer (gen);
21731 size_t loh_pad = AlignQword (loh_padding_obj_size);
21733 generation_allocation_pointer (gen) += size + loh_pad;
21734 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
21736 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
21737 generation_allocation_pointer (gen),
21738 generation_allocation_limit (gen),
21739 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21741 assert (result + loh_pad);
21742 return result + loh_pad;
21746 BOOL gc_heap::loh_compaction_requested()
21748 // If hard limit is specified GC will automatically decide if LOH needs to be compacted.
21749 return (loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
21753 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
21755 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
21757 if (all_heaps_compacted_p)
21759 // If the compaction mode says to compact once and we are going to compact LOH,
21760 // we need to revert it back to no compaction.
21761 loh_compaction_mode = loh_compaction_default;
21766 BOOL gc_heap::plan_loh()
21768 if (!loh_pinned_queue)
21770 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
21771 if (!loh_pinned_queue)
21773 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
21774 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
21778 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
21781 if (heap_number == 0)
21782 loh_pinned_queue_decay = LOH_PIN_DECAY;
21784 loh_pinned_queue_tos = 0;
21785 loh_pinned_queue_bos = 0;
21787 generation* gen = large_object_generation;
21788 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21789 PREFIX_ASSUME(start_seg != NULL);
21790 heap_segment* seg = start_seg;
21791 uint8_t* o = generation_allocation_start (gen);
21793 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
21794 generation_size (loh_generation),
21795 generation_free_list_space (gen),
21796 generation_free_obj_space (gen)));
21800 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
21801 seg = heap_segment_next (seg);
21806 //Skip the generation gap object
21807 o = o + AlignQword (size (o));
21808 // We don't need to ever realloc gen3 start so don't touch it.
21809 heap_segment_plan_allocated (seg) = o;
21810 generation_allocation_pointer (gen) = o;
21811 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21812 generation_allocation_segment (gen) = start_seg;
21814 uint8_t* free_space_start = o;
21815 uint8_t* free_space_end = o;
21816 uint8_t* new_address = 0;
21820 if (o >= heap_segment_allocated (seg))
21822 seg = heap_segment_next (seg);
21828 o = heap_segment_mem (seg);
21833 free_space_end = o;
21834 size_t size = AlignQword (size (o));
21835 dprintf (1235, ("%Ix(%Id) M", o, size));
21839 // We don't clear the pinned bit yet so we can check in
21840 // compact phase how big a free object we should allocate
21841 // in front of the pinned object. We use the reloc address
21842 // field to store this.
21843 if (!loh_enque_pinned_plug (o, size))
21851 new_address = loh_allocate_in_condemned (size);
21854 loh_set_node_relocation_distance (o, (new_address - o));
21855 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
21858 free_space_start = o;
21859 if (o < heap_segment_allocated (seg))
21861 assert (!marked (o));
21866 while (o < heap_segment_allocated (seg) && !marked (o))
21868 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
21869 o = o + AlignQword (size (o));
21874 while (!loh_pinned_plug_que_empty_p())
21876 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21877 size_t len = pinned_len (m);
21878 uint8_t* plug = pinned_plug (m);
21880 // detect pinned block in different segment (later) than
21881 // allocation segment
21882 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
21884 while ((plug < generation_allocation_pointer (gen)) ||
21885 (plug >= heap_segment_allocated (nseg)))
21887 assert ((plug < heap_segment_mem (nseg)) ||
21888 (plug > heap_segment_reserved (nseg)));
21889 //adjust the end of the segment to be the end of the plug
21890 assert (generation_allocation_pointer (gen)>=
21891 heap_segment_mem (nseg));
21892 assert (generation_allocation_pointer (gen)<=
21893 heap_segment_committed (nseg));
21895 heap_segment_plan_allocated (nseg) =
21896 generation_allocation_pointer (gen);
21897 //switch allocation segment
21898 nseg = heap_segment_next_rw (nseg);
21899 generation_allocation_segment (gen) = nseg;
21900 //reset the allocation pointer and limits
21901 generation_allocation_pointer (gen) =
21902 heap_segment_mem (nseg);
21905 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21906 pinned_len (m) = plug - generation_allocation_pointer (gen);
21907 generation_allocation_pointer (gen) = plug + len;
21910 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21911 generation_allocation_pointer (gen) = 0;
21912 generation_allocation_limit (gen) = 0;
21917 void gc_heap::compact_loh()
21919 assert (loh_compaction_requested() || heap_hard_limit);
21921 generation* gen = large_object_generation;
21922 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21923 PREFIX_ASSUME(start_seg != NULL);
21924 heap_segment* seg = start_seg;
21925 heap_segment* prev_seg = 0;
21926 uint8_t* o = generation_allocation_start (gen);
21928 //Skip the generation gap object
21929 o = o + AlignQword (size (o));
21930 // We don't need to ever realloc gen3 start so don't touch it.
21931 uint8_t* free_space_start = o;
21932 uint8_t* free_space_end = o;
21933 generation_allocator (gen)->clear();
21934 generation_free_list_space (gen) = 0;
21935 generation_free_obj_space (gen) = 0;
21937 loh_pinned_queue_bos = 0;
21941 if (o >= heap_segment_allocated (seg))
21943 heap_segment* next_seg = heap_segment_next (seg);
21945 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21946 (seg != start_seg) && !heap_segment_read_only_p (seg))
21948 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21950 heap_segment_next (prev_seg) = next_seg;
21951 heap_segment_next (seg) = freeable_uoh_segment;
21952 freeable_uoh_segment = seg;
21956 if (!heap_segment_read_only_p (seg))
21958 // We grew the segment to accommodate allocations.
21959 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21961 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21963 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21967 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21968 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21969 decommit_heap_segment_pages (seg, 0);
21970 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21972 heap_segment_allocated (seg),
21973 heap_segment_used (seg),
21974 heap_segment_committed (seg)));
21975 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21976 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21986 o = heap_segment_mem (seg);
21992 free_space_end = o;
21993 size_t size = AlignQword (size (o));
21996 uint8_t* reloc = o;
22001 // We are relying on the fact the pinned objects are always looked at in the same order
22002 // in plan phase and in compact phase.
22003 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
22004 uint8_t* plug = pinned_plug (m);
22005 assert (plug == o);
22007 loh_pad = pinned_len (m);
22012 loh_pad = AlignQword (loh_padding_obj_size);
22014 reloc += loh_node_relocation_distance (o);
22015 gcmemcopy (reloc, o, size, TRUE);
22018 thread_gap ((reloc - loh_pad), loh_pad, gen);
22021 free_space_start = o;
22022 if (o < heap_segment_allocated (seg))
22024 assert (!marked (o));
22029 while (o < heap_segment_allocated (seg) && !marked (o))
22031 o = o + AlignQword (size (o));
22036 assert (loh_pinned_plug_que_empty_p());
22038 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
22039 generation_size (loh_generation),
22040 generation_free_list_space (gen),
22041 generation_free_obj_space (gen)));
22044 void gc_heap::relocate_in_loh_compact()
22046 generation* gen = large_object_generation;
22047 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
22048 uint8_t* o = generation_allocation_start (gen);
22050 //Skip the generation gap object
22051 o = o + AlignQword (size (o));
22055 if (o >= heap_segment_allocated (seg))
22057 seg = heap_segment_next (seg);
22063 o = heap_segment_mem (seg);
22068 size_t size = AlignQword (size (o));
22070 check_class_object_demotion (o);
22071 if (contain_pointers (o))
22073 go_through_object_nostart (method_table (o), o, size(o), pval,
22075 reloc_survivor_helper (pval);
22080 if (o < heap_segment_allocated (seg))
22082 assert (!marked (o));
22087 while (o < heap_segment_allocated (seg) && !marked (o))
22089 o = o + AlignQword (size (o));
22094 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
22095 generation_size (loh_generation),
22096 generation_free_list_space (gen),
22097 generation_free_obj_space (gen)));
22100 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
22102 generation* gen = large_object_generation;
22103 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
22104 uint8_t* o = generation_allocation_start (gen);
22106 //Skip the generation gap object
22107 o = o + AlignQword (size (o));
22111 if (o >= heap_segment_allocated (seg))
22113 seg = heap_segment_next (seg);
22119 o = heap_segment_mem (seg);
22124 size_t size = AlignQword (size (o));
22126 ptrdiff_t reloc = loh_node_relocation_distance (o);
22128 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
22130 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
22133 if (o < heap_segment_allocated (seg))
22135 assert (!marked (o));
22140 while (o < heap_segment_allocated (seg) && !marked (o))
22142 o = o + AlignQword (size (o));
22148 BOOL gc_heap::loh_object_p (uint8_t* o)
22150 #ifdef MULTIPLE_HEAPS
22151 gc_heap* hp = gc_heap::g_heaps [0];
22152 int brick_entry = hp->brick_table[hp->brick_of (o)];
22153 #else //MULTIPLE_HEAPS
22154 int brick_entry = brick_table[brick_of (o)];
22155 #endif //MULTIPLE_HEAPS
22157 return (brick_entry == 0);
22159 #endif //FEATURE_LOH_COMPACTION
22161 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
22162 BOOL& last_pinned_plug_p,
22163 BOOL& pinned_plug_p,
22165 size_t& artificial_pinned_size)
22167 last_npinned_plug_p = FALSE;
22168 last_pinned_plug_p = TRUE;
22169 pinned_plug_p = TRUE;
22170 artificial_pinned_size = ps;
22173 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
22174 // plugs are always interleaved.
22175 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
22177 BOOL& last_npinned_plug_p,
22178 BOOL& last_pinned_plug_p,
22179 uint8_t*& last_pinned_plug,
22180 BOOL& pinned_plug_p,
22181 uint8_t* last_object_in_last_plug,
22182 BOOL& merge_with_last_pin_p,
22183 // this is only for verification purpose
22184 size_t last_plug_len)
22186 UNREFERENCED_PARAMETER(last_plug_len);
22188 if (!last_npinned_plug_p && !last_pinned_plug_p)
22190 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
22191 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
22192 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
22193 set_gap_size (plug_start, plug_start - plug_end);
22196 if (pinned (plug_start))
22198 BOOL save_pre_plug_info_p = FALSE;
22200 if (last_npinned_plug_p || last_pinned_plug_p)
22202 //if (last_plug_len == Align (min_obj_size))
22204 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
22205 // GCToOSInterface::DebugBreak();
22207 save_pre_plug_info_p = TRUE;
22210 pinned_plug_p = TRUE;
22211 last_npinned_plug_p = FALSE;
22213 if (last_pinned_plug_p)
22215 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
22216 merge_with_last_pin_p = TRUE;
22220 last_pinned_plug_p = TRUE;
22221 last_pinned_plug = plug_start;
22223 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
22225 if (save_pre_plug_info_p)
22227 set_gap_size (plug_start, sizeof (gap_reloc_pair));
22233 if (last_pinned_plug_p)
22235 //if (Align (last_plug_len) < min_pre_pin_obj_size)
22237 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
22238 // GCToOSInterface::DebugBreak();
22241 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
22242 set_gap_size (plug_start, sizeof (gap_reloc_pair));
22244 verify_pins_with_post_plug_info("after saving post plug info");
22246 last_npinned_plug_p = TRUE;
22247 last_pinned_plug_p = FALSE;
22251 void gc_heap::record_interesting_data_point (interesting_data_point idp)
22253 #ifdef GC_CONFIG_DRIVEN
22254 (interesting_data_per_gc[idp])++;
22256 UNREFERENCED_PARAMETER(idp);
22257 #endif //GC_CONFIG_DRIVEN
22261 #pragma warning(push)
22262 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
22264 void gc_heap::plan_phase (int condemned_gen_number)
22266 size_t old_gen2_allocated = 0;
22267 size_t old_gen2_size = 0;
22269 if (condemned_gen_number == (max_generation - 1))
22271 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
22272 old_gen2_size = generation_size (max_generation);
22275 assert (settings.concurrent == FALSE);
22277 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
22278 condemned_gen_number, settings.promotion ? 1 : 0));
22280 generation* condemned_gen1 = generation_of (condemned_gen_number);
22283 BOOL use_mark_list = FALSE;
22284 uint8_t** mark_list_next = &mark_list[0];
22285 #ifdef GC_CONFIG_DRIVEN
22286 dprintf (3, ("total number of marked objects: %Id (%Id)",
22287 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
22289 if (mark_list_index >= (mark_list_end + 1))
22291 mark_list_index = mark_list_end + 1;
22292 #ifndef MULTIPLE_HEAPS // in Server GC, we check for mark list overflow in sort_mark_list
22293 mark_list_overflow = true;
22297 dprintf (3, ("mark_list length: %Id",
22298 (mark_list_index - &mark_list[0])));
22299 #endif //GC_CONFIG_DRIVEN
22301 if ((condemned_gen_number < max_generation) &&
22302 (mark_list_index <= mark_list_end)
22303 #ifdef BACKGROUND_GC
22304 && (!gc_heap::background_running_p())
22305 #endif //BACKGROUND_GC
22308 #ifndef MULTIPLE_HEAPS
22310 do_vxsort (mark_list, mark_list_index - mark_list, slow, shigh);
22312 _sort (&mark_list[0], mark_list_index - 1, 0);
22313 #endif //USE_VXSORT
22315 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
22316 //verify_qsort_array (&mark_list[0], mark_list_index-1);
22317 #endif //!MULTIPLE_HEAPS
22318 use_mark_list = TRUE;
22319 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
22323 dprintf (3, ("mark_list not used"));
22328 #ifdef FEATURE_BASICFREEZE
22329 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
22330 ro_segments_in_range)
22332 sweep_ro_segments (generation_start_segment (condemned_gen1));
22334 #endif // FEATURE_BASICFREEZE
22336 #ifndef MULTIPLE_HEAPS
22337 if (shigh != (uint8_t*)0)
22339 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
22341 PREFIX_ASSUME(seg != NULL);
22343 heap_segment* fseg = seg;
22346 if (slow > heap_segment_mem (seg) &&
22347 slow < heap_segment_reserved (seg))
22351 uint8_t* o = generation_allocation_start (condemned_gen1) +
22352 Align (size (generation_allocation_start (condemned_gen1)));
22355 assert ((slow - o) >= (int)Align (min_obj_size));
22356 #ifdef BACKGROUND_GC
22357 if (current_c_gc_state == c_gc_state_marking)
22359 bgc_clear_batch_mark_array_bits (o, slow);
22361 #endif //BACKGROUND_GC
22362 make_unused_array (o, slow - o);
22367 assert (condemned_gen_number == max_generation);
22368 make_unused_array (heap_segment_mem (seg),
22369 slow - heap_segment_mem (seg));
22372 if (in_range_for_segment (shigh, seg))
22374 #ifdef BACKGROUND_GC
22375 if (current_c_gc_state == c_gc_state_marking)
22377 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
22379 #endif //BACKGROUND_GC
22380 heap_segment_allocated (seg) = shigh + Align (size (shigh));
22382 // test if the segment is in the range of [slow, shigh]
22383 if (!((heap_segment_reserved (seg) >= slow) &&
22384 (heap_segment_mem (seg) <= shigh)))
22386 // shorten it to minimum
22387 heap_segment_allocated (seg) = heap_segment_mem (seg);
22389 seg = heap_segment_next_rw (seg);
22394 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
22396 PREFIX_ASSUME(seg != NULL);
22398 heap_segment* sseg = seg;
22401 // shorten it to minimum
22404 // no survivors make all generations look empty
22405 uint8_t* o = generation_allocation_start (condemned_gen1) +
22406 Align (size (generation_allocation_start (condemned_gen1)));
22407 #ifdef BACKGROUND_GC
22408 if (current_c_gc_state == c_gc_state_marking)
22410 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
22412 #endif //BACKGROUND_GC
22413 heap_segment_allocated (seg) = o;
22417 assert (condemned_gen_number == max_generation);
22418 #ifdef BACKGROUND_GC
22419 if (current_c_gc_state == c_gc_state_marking)
22421 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
22423 #endif //BACKGROUND_GC
22424 heap_segment_allocated (seg) = heap_segment_mem (seg);
22426 seg = heap_segment_next_rw (seg);
22430 #endif //MULTIPLE_HEAPS
22432 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
22434 PREFIX_ASSUME(seg1 != NULL);
22436 uint8_t* end = heap_segment_allocated (seg1);
22437 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
22438 uint8_t* x = first_condemned_address;
22440 assert (!marked (x));
22441 uint8_t* plug_end = x;
22443 size_t sequence_number = 0;
22444 uint8_t* last_node = 0;
22445 size_t current_brick = brick_of (x);
22446 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
22447 (settings.promotion == FALSE));
22448 int active_old_gen_number = condemned_gen_number;
22449 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
22450 (1 + condemned_gen_number));
22451 generation* older_gen = 0;
22452 generation* consing_gen = condemned_gen1;
22453 alloc_list r_free_list [MAX_SOH_BUCKET_COUNT];
22455 size_t r_free_list_space = 0;
22456 size_t r_free_obj_space = 0;
22457 size_t r_older_gen_free_list_allocated = 0;
22458 size_t r_older_gen_condemned_allocated = 0;
22459 size_t r_older_gen_end_seg_allocated = 0;
22460 uint8_t* r_allocation_pointer = 0;
22461 uint8_t* r_allocation_limit = 0;
22462 uint8_t* r_allocation_start_region = 0;
22463 heap_segment* r_allocation_segment = 0;
22464 #ifdef FREE_USAGE_STATS
22465 size_t r_older_gen_free_space[NUM_GEN_POWER2];
22466 #endif //FREE_USAGE_STATS
22468 if ((condemned_gen_number < max_generation))
22470 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
22471 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
22473 r_free_list_space = generation_free_list_space (older_gen);
22474 r_free_obj_space = generation_free_obj_space (older_gen);
22475 #ifdef FREE_USAGE_STATS
22476 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
22477 #endif //FREE_USAGE_STATS
22478 generation_allocate_end_seg_p (older_gen) = FALSE;
22479 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
22480 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
22481 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
22482 r_allocation_limit = generation_allocation_limit (older_gen);
22483 r_allocation_pointer = generation_allocation_pointer (older_gen);
22484 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
22485 r_allocation_segment = generation_allocation_segment (older_gen);
22486 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
22488 PREFIX_ASSUME(start_seg != NULL);
22490 if (start_seg != ephemeral_heap_segment)
22492 assert (condemned_gen_number == (max_generation - 1));
22493 while (start_seg && (start_seg != ephemeral_heap_segment))
22495 assert (heap_segment_allocated (start_seg) >=
22496 heap_segment_mem (start_seg));
22497 assert (heap_segment_allocated (start_seg) <=
22498 heap_segment_reserved (start_seg));
22499 heap_segment_plan_allocated (start_seg) =
22500 heap_segment_allocated (start_seg);
22501 start_seg = heap_segment_next_rw (start_seg);
22506 //reset all of the segment allocated sizes
22508 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
22510 PREFIX_ASSUME(seg2 != NULL);
22514 heap_segment_plan_allocated (seg2) =
22515 heap_segment_mem (seg2);
22516 seg2 = heap_segment_next_rw (seg2);
22519 int condemned_gn = condemned_gen_number;
22521 int bottom_gen = 0;
22522 init_free_and_plug();
22524 while (condemned_gn >= bottom_gen)
22526 generation* condemned_gen2 = generation_of (condemned_gn);
22527 generation_allocator (condemned_gen2)->clear();
22528 generation_free_list_space (condemned_gen2) = 0;
22529 generation_free_obj_space (condemned_gen2) = 0;
22530 generation_allocation_size (condemned_gen2) = 0;
22531 generation_condemned_allocated (condemned_gen2) = 0;
22532 generation_sweep_allocated (condemned_gen2) = 0;
22533 generation_pinned_allocated (condemned_gen2) = 0;
22534 generation_free_list_allocated(condemned_gen2) = 0;
22535 generation_end_seg_allocated (condemned_gen2) = 0;
22536 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
22537 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
22538 #ifdef FREE_USAGE_STATS
22539 generation_pinned_free_obj_space (condemned_gen2) = 0;
22540 generation_allocated_in_pinned_free (condemned_gen2) = 0;
22541 generation_allocated_since_last_pin (condemned_gen2) = 0;
22542 #endif //FREE_USAGE_STATS
22543 generation_plan_allocation_start (condemned_gen2) = 0;
22544 generation_allocation_segment (condemned_gen2) =
22545 heap_segment_rw (generation_start_segment (condemned_gen2));
22547 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
22549 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
22551 generation_allocation_pointer (condemned_gen2) =
22552 heap_segment_mem (generation_allocation_segment (condemned_gen2));
22556 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
22559 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22560 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22565 BOOL allocate_first_generation_start = FALSE;
22567 if (allocate_in_condemned)
22569 allocate_first_generation_start = TRUE;
22572 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22574 demotion_low = MAX_PTR;
22575 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
22577 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
22578 // from gen1. They should get promoted to gen2.
22579 demote_gen1_p = !(settings.promotion &&
22580 (settings.condemned_generation == (max_generation - 1)) &&
22581 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
22583 total_ephemeral_size = 0;
22585 print_free_and_plug ("BP");
22587 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22589 generation* temp_gen = generation_of (gen_idx);
22591 dprintf (2, ("gen%d start %Ix, plan start %Ix",
22593 generation_allocation_start (temp_gen),
22594 generation_plan_allocation_start (temp_gen)));
22597 BOOL fire_pinned_plug_events_p = EVENT_ENABLED(PinPlugAtGCTime);
22598 size_t last_plug_len = 0;
22604 if (!use_mark_list)
22608 assert (heap_segment_allocated (seg1) == end);
22609 heap_segment_allocated (seg1) = plug_end;
22611 current_brick = update_brick_table (tree, current_brick, x, plug_end);
22612 dprintf (3, ("end of seg: new tree, sequence# 0"));
22613 sequence_number = 0;
22616 if (heap_segment_next_rw (seg1))
22618 seg1 = heap_segment_next_rw (seg1);
22619 end = heap_segment_allocated (seg1);
22620 plug_end = x = heap_segment_mem (seg1);
22621 current_brick = brick_of (x);
22622 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22631 BOOL last_npinned_plug_p = FALSE;
22632 BOOL last_pinned_plug_p = FALSE;
22634 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
22635 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
22636 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
22637 uint8_t* last_pinned_plug = 0;
22638 size_t num_pinned_plugs_in_plug = 0;
22640 uint8_t* last_object_in_plug = 0;
22642 while ((x < end) && marked (x))
22644 uint8_t* plug_start = x;
22645 uint8_t* saved_plug_end = plug_end;
22646 BOOL pinned_plug_p = FALSE;
22647 BOOL npin_before_pin_p = FALSE;
22648 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
22649 uint8_t* saved_last_object_in_plug = last_object_in_plug;
22650 BOOL merge_with_last_pin_p = FALSE;
22652 size_t added_pinning_size = 0;
22653 size_t artificial_pinned_size = 0;
22655 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
22656 last_pinned_plug, pinned_plug_p, last_object_in_plug,
22657 merge_with_last_pin_p, last_plug_len);
22659 #ifdef FEATURE_STRUCTALIGN
22660 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
22661 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
22662 #endif // FEATURE_STRUCTALIGN
22666 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
22673 #ifdef FEATURE_STRUCTALIGN
22676 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
22677 if (obj_requiredAlignment > requiredAlignment)
22679 requiredAlignment = obj_requiredAlignment;
22680 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
22683 #endif // FEATURE_STRUCTALIGN
22687 dprintf(4, ("+%Ix+", (size_t)xl));
22688 assert ((size (xl) > 0));
22689 assert ((size (xl) <= loh_size_threshold));
22691 last_object_in_plug = xl;
22693 xl = xl + Align (size (xl));
22697 BOOL next_object_marked_p = ((xl < end) && marked (xl));
22701 // If it is pinned we need to extend to the next marked object as we can't use part of
22702 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
22703 // references but for now I am just using the next non pinned object for that).
22704 if (next_object_marked_p)
22707 last_object_in_plug = xl;
22708 size_t extra_size = Align (size (xl));
22709 xl = xl + extra_size;
22710 added_pinning_size = extra_size;
22715 if (next_object_marked_p)
22716 npin_before_pin_p = TRUE;
22719 assert (xl <= end);
22722 dprintf (3, ( "%Ix[", (size_t)x));
22724 size_t ps = plug_end - plug_start;
22725 last_plug_len = ps;
22726 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
22727 uint8_t* new_address = 0;
22729 if (!pinned_plug_p)
22731 if (allocate_in_condemned &&
22732 (settings.condemned_generation == max_generation) &&
22733 (ps > OS_PAGE_SIZE))
22735 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
22736 //reloc should >=0 except when we relocate
22737 //across segments and the dest seg is higher then the src
22739 if ((ps > (8*OS_PAGE_SIZE)) &&
22741 ((size_t)reloc < (ps/16)))
22743 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
22744 (size_t)plug_start, reloc));
22745 // The last plug couldn't have been a npinned plug or it would have
22746 // included this plug.
22747 assert (!saved_last_npinned_plug_p);
22749 if (last_pinned_plug)
22751 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
22752 merge_with_last_pin_p = TRUE;
22756 enque_pinned_plug (plug_start, FALSE, 0);
22757 last_pinned_plug = plug_start;
22760 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22761 ps, artificial_pinned_size);
22766 if (allocate_first_generation_start)
22768 allocate_first_generation_start = FALSE;
22769 plan_generation_start (condemned_gen1, consing_gen, plug_start);
22770 assert (generation_plan_allocation_start (condemned_gen1));
22773 if (seg1 == ephemeral_heap_segment)
22775 process_ephemeral_boundaries (plug_start, active_new_gen_number,
22776 active_old_gen_number,
22778 allocate_in_condemned);
22781 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
22783 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
22784 dd_survived_size (dd_active_old) += ps;
22786 BOOL convert_to_pinned_p = FALSE;
22788 if (!pinned_plug_p)
22790 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
22791 dd_num_npinned_plugs (dd_active_old)++;
22792 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
22794 add_gen_plug (active_old_gen_number, ps);
22796 if (allocate_in_condemned)
22798 verify_pins_with_post_plug_info("before aic");
22801 allocate_in_condemned_generations (consing_gen,
22803 active_old_gen_number,
22805 &convert_to_pinned_p,
22806 (npin_before_pin_p ? plug_end : 0),
22808 #endif //SHORT_PLUGS
22809 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22810 verify_pins_with_post_plug_info("after aic");
22814 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
22816 if (new_address != 0)
22818 if (settings.condemned_generation == (max_generation - 1))
22820 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
22821 plug_start, plug_end,
22822 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
22823 (size_t)(plug_end - plug_start)));
22828 if (generation_allocator(older_gen)->discard_if_no_fit_p())
22830 allocate_in_condemned = TRUE;
22833 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
22835 &convert_to_pinned_p,
22836 (npin_before_pin_p ? plug_end : 0),
22838 #endif //SHORT_PLUGS
22839 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22843 if (convert_to_pinned_p)
22845 assert (last_npinned_plug_p != FALSE);
22846 assert (last_pinned_plug_p == FALSE);
22847 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22848 ps, artificial_pinned_size);
22849 enque_pinned_plug (plug_start, FALSE, 0);
22850 last_pinned_plug = plug_start;
22856 //verify that we are at then end of the ephemeral segment
22857 assert (generation_allocation_segment (consing_gen) ==
22858 ephemeral_heap_segment);
22859 //verify that we are near the end
22860 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
22861 heap_segment_allocated (ephemeral_heap_segment));
22862 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
22863 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
22867 #ifdef SIMPLE_DPRINTF
22868 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
22869 (size_t)(node_gap_size (plug_start)),
22870 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
22871 (size_t)new_address + ps, ps,
22872 (is_plug_padded (plug_start) ? 1 : 0)));
22873 #endif //SIMPLE_DPRINTF
22876 if (is_plug_padded (plug_start))
22878 dprintf (3, ("%Ix was padded", plug_start));
22879 dd_padding_size (dd_active_old) += Align (min_obj_size);
22881 #endif //SHORT_PLUGS
22888 if (fire_pinned_plug_events_p)
22890 FIRE_EVENT(PinPlugAtGCTime, plug_start, plug_end,
22891 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)));
22894 if (merge_with_last_pin_p)
22896 merge_with_last_pinned_plug (last_pinned_plug, ps);
22900 assert (last_pinned_plug == plug_start);
22901 set_pinned_info (plug_start, ps, consing_gen);
22904 new_address = plug_start;
22906 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22907 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22908 (size_t)plug_end, ps,
22909 (merge_with_last_pin_p ? 1 : 0)));
22911 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22912 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22913 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22914 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22916 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22918 last_gen1_pin_end = plug_end;
22923 // detect forward allocation in the same segment
22924 assert (!((new_address > plug_start) &&
22925 (new_address < heap_segment_reserved (seg1))));
22928 if (!merge_with_last_pin_p)
22930 if (current_brick != brick_of (plug_start))
22932 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22933 sequence_number = 0;
22937 set_node_relocation_distance (plug_start, (new_address - plug_start));
22938 if (last_node && (node_relocation_distance (last_node) ==
22939 (node_relocation_distance (plug_start) +
22940 (ptrdiff_t)node_gap_size (plug_start))))
22942 //dprintf(3,( " Lb"));
22943 dprintf (3, ("%Ix Lb", plug_start));
22944 set_node_left (plug_start);
22946 if (0 == sequence_number)
22948 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22952 verify_pins_with_post_plug_info("before insert node");
22954 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22955 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22956 last_node = plug_start;
22959 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22960 if (!pinned_plug_p)
22962 if (mark_stack_tos > 0)
22964 mark& m = mark_stack_array[mark_stack_tos - 1];
22965 if (m.has_post_plug_info())
22967 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22968 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22969 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22971 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22972 *current_plug_gap_start, *(current_plug_gap_start + 1),
22973 *(current_plug_gap_start + 2)));
22974 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22981 verify_pins_with_post_plug_info("after insert node");
22985 if (num_pinned_plugs_in_plug > 1)
22987 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22994 while ((mark_list_next < mark_list_index) &&
22995 (*mark_list_next <= x))
22999 if ((mark_list_next < mark_list_index)
23000 #ifdef MULTIPLE_HEAPS
23001 && (*mark_list_next < end) //for multiple segments
23002 #endif //MULTIPLE_HEAPS
23004 x = *mark_list_next;
23012 #ifdef BACKGROUND_GC
23013 if (current_c_gc_state == c_gc_state_marking)
23015 assert (gc_heap::background_running_p());
23016 while ((xl < end) && !marked (xl))
23018 dprintf (4, ("-%Ix-", (size_t)xl));
23019 assert ((size (xl) > 0));
23020 background_object_marked (xl, TRUE);
23021 xl = xl + Align (size (xl));
23026 #endif //BACKGROUND_GC
23028 while ((xl < end) && !marked (xl))
23030 dprintf (4, ("-%Ix-", (size_t)xl));
23031 assert ((size (xl) > 0));
23032 xl = xl + Align (size (xl));
23036 assert (xl <= end);
23042 while (!pinned_plug_que_empty_p())
23044 if (settings.promotion)
23046 uint8_t* pplug = pinned_plug (oldest_pin());
23047 if (in_range_for_segment (pplug, ephemeral_heap_segment))
23049 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
23050 //allocate all of the generation gaps
23051 while (active_new_gen_number > 0)
23053 active_new_gen_number--;
23055 if (active_new_gen_number == (max_generation - 1))
23057 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
23058 if (!demote_gen1_p)
23059 advance_pins_for_demotion (consing_gen);
23062 generation* gen = generation_of (active_new_gen_number);
23063 plan_generation_start (gen, consing_gen, 0);
23065 if (demotion_low == MAX_PTR)
23067 demotion_low = pplug;
23068 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
23071 dprintf (2, ("(%d)gen%d plan start: %Ix",
23072 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
23073 assert (generation_plan_allocation_start (gen));
23078 if (pinned_plug_que_empty_p())
23081 size_t entry = deque_pinned_plug();
23082 mark* m = pinned_plug_of (entry);
23083 uint8_t* plug = pinned_plug (m);
23084 size_t len = pinned_len (m);
23086 // detect pinned block in different segment (later) than
23087 // allocation segment
23088 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
23090 while ((plug < generation_allocation_pointer (consing_gen)) ||
23091 (plug >= heap_segment_allocated (nseg)))
23093 assert ((plug < heap_segment_mem (nseg)) ||
23094 (plug > heap_segment_reserved (nseg)));
23095 //adjust the end of the segment to be the end of the plug
23096 assert (generation_allocation_pointer (consing_gen)>=
23097 heap_segment_mem (nseg));
23098 assert (generation_allocation_pointer (consing_gen)<=
23099 heap_segment_committed (nseg));
23101 heap_segment_plan_allocated (nseg) =
23102 generation_allocation_pointer (consing_gen);
23103 //switch allocation segment
23104 nseg = heap_segment_next_rw (nseg);
23105 generation_allocation_segment (consing_gen) = nseg;
23106 //reset the allocation pointer and limits
23107 generation_allocation_pointer (consing_gen) =
23108 heap_segment_mem (nseg);
23111 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
23112 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
23113 (size_t)(brick_table[brick_of (plug)])));
23115 generation_allocation_pointer (consing_gen) = plug + len;
23116 generation_allocation_limit (consing_gen) =
23117 generation_allocation_pointer (consing_gen);
23118 //Add the size of the pinned plug to the right pinned allocations
23119 //find out which gen this pinned plug came from
23120 int frgn = object_gennum (plug);
23121 if ((frgn != (int)max_generation) && settings.promotion)
23123 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
23128 plan_generation_starts (consing_gen);
23129 print_free_and_plug ("AP");
23132 #ifdef SIMPLE_DPRINTF
23133 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
23135 generation* temp_gen = generation_of (gen_idx);
23136 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
23138 int added_pinning_ratio = 0;
23139 int artificial_pinned_ratio = 0;
23141 if (dd_pinned_survived_size (temp_dd) != 0)
23143 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
23144 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
23147 size_t padding_size =
23149 dd_padding_size (temp_dd);
23152 #endif //SHORT_PLUGS
23153 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
23155 generation_allocation_start (temp_gen),
23156 generation_plan_allocation_start (temp_gen),
23157 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
23158 generation_allocation_size (temp_gen),
23159 generation_pinned_allocation_compact_size (temp_gen),
23160 generation_pinned_allocation_sweep_size (temp_gen),
23161 dd_survived_size (temp_dd),
23162 dd_pinned_survived_size (temp_dd),
23163 added_pinning_ratio,
23164 artificial_pinned_ratio,
23165 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
23168 #endif //SIMPLE_DPRINTF
23171 if (settings.condemned_generation == (max_generation - 1 ))
23173 generation* older_gen = generation_of (settings.condemned_generation + 1);
23174 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
23175 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
23176 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
23177 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
23179 size_t growth = end_seg_allocated + condemned_allocated;
23183 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, condemned alloc: %Id",
23184 growth, end_seg_allocated, condemned_allocated));
23186 maxgen_size_inc_p = true;
23190 dprintf (2, ("gen2 didn't grow (end seg alloc: %Id, , condemned alloc: %Id, gen1 c alloc: %Id",
23191 end_seg_allocated, condemned_allocated,
23192 generation_condemned_allocated (generation_of (max_generation - 1))));
23195 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
23196 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
23197 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
23198 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
23200 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected)",
23201 free_list_allocated, rejected_free_space));
23203 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
23204 maxgen_size_info->free_list_allocated = free_list_allocated;
23205 maxgen_size_info->free_list_rejected = rejected_free_space;
23206 maxgen_size_info->end_seg_allocated = end_seg_allocated;
23207 maxgen_size_info->condemned_allocated = condemned_allocated;
23208 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
23209 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
23211 #ifdef FREE_USAGE_STATS
23212 int free_list_efficiency = 0;
23213 if ((free_list_allocated + rejected_free_space) != 0)
23214 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
23216 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
23218 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
23219 older_gen->gen_num,
23220 free_list_efficiency, running_free_list_efficiency));
23222 dprintf (1, ("gen2 free list change"));
23223 for (int j = 0; j < NUM_GEN_POWER2; j++)
23225 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
23228 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
23229 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
23230 (generation_of(max_generation - 1))->gen_plugs[j]));
23232 #endif //FREE_USAGE_STATS
23235 size_t fragmentation =
23236 generation_fragmentation (generation_of (condemned_gen_number),
23238 heap_segment_allocated (ephemeral_heap_segment));
23240 dprintf (2,("Fragmentation: %Id", fragmentation));
23241 dprintf (2,("---- End of Plan phase ----"));
23243 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
23244 assert(IsGCInProgress());
23246 BOOL should_expand = FALSE;
23247 BOOL should_compact= FALSE;
23248 ephemeral_promotion = FALSE;
23251 if ((!settings.concurrent) &&
23252 !provisional_mode_triggered &&
23253 ((condemned_gen_number < max_generation) &&
23254 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
23256 dprintf (GTC_LOG, ("gen0 reduction count is %d, condemning %d, mem load %d",
23257 settings.gen0_reduction_count,
23258 condemned_gen_number,
23259 settings.entry_memory_load));
23260 should_compact = TRUE;
23262 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
23263 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
23265 if ((condemned_gen_number >= (max_generation - 1)) &&
23266 dt_low_ephemeral_space_p (tuning_deciding_expansion))
23268 dprintf (GTC_LOG, ("Not enough space for all ephemeral generations with compaction"));
23269 should_expand = TRUE;
23274 #endif // HOST_64BIT
23275 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
23278 #endif // HOST_64BIT
23280 #ifdef FEATURE_LOH_COMPACTION
23281 loh_compacted_p = FALSE;
23282 #endif //FEATURE_LOH_COMPACTION
23284 if (condemned_gen_number == max_generation)
23286 #ifdef FEATURE_LOH_COMPACTION
23287 if (settings.loh_compaction)
23291 should_compact = TRUE;
23292 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
23293 loh_compacted_p = TRUE;
23298 if ((heap_number == 0) && (loh_pinned_queue))
23300 loh_pinned_queue_decay--;
23302 if (!loh_pinned_queue_decay)
23304 delete loh_pinned_queue;
23305 loh_pinned_queue = 0;
23310 if (!loh_compacted_p)
23311 #endif //FEATURE_LOH_COMPACTION
23313 GCToEEInterface::DiagWalkUOHSurvivors(__this, loh_generation);
23314 sweep_uoh_objects (loh_generation);
23317 GCToEEInterface::DiagWalkUOHSurvivors(__this, poh_generation);
23318 sweep_uoh_objects (poh_generation);
23322 settings.loh_compaction = FALSE;
23325 #ifdef MULTIPLE_HEAPS
23327 new_heap_segment = NULL;
23329 if (should_compact && should_expand)
23330 gc_policy = policy_expand;
23331 else if (should_compact)
23332 gc_policy = policy_compact;
23334 gc_policy = policy_sweep;
23336 //vote for result of should_compact
23337 dprintf (3, ("Joining for compaction decision"));
23338 gc_t_join.join(this, gc_join_decide_on_compaction);
23339 if (gc_t_join.joined())
23341 //safe place to delete large heap segments
23342 if (condemned_gen_number == max_generation)
23344 for (int i = 0; i < n_heaps; i++)
23346 g_heaps [i]->rearrange_uoh_segments ();
23350 if (maxgen_size_inc_p && provisional_mode_triggered)
23352 pm_trigger_full_gc = true;
23353 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
23357 settings.demotion = FALSE;
23358 int pol_max = policy_sweep;
23359 #ifdef GC_CONFIG_DRIVEN
23360 BOOL is_compaction_mandatory = FALSE;
23361 #endif //GC_CONFIG_DRIVEN
23364 for (i = 0; i < n_heaps; i++)
23366 if (pol_max < g_heaps[i]->gc_policy)
23367 pol_max = policy_compact;
23368 // set the demotion flag is any of the heap has demotion
23369 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
23371 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
23372 settings.demotion = TRUE;
23375 #ifdef GC_CONFIG_DRIVEN
23376 if (!is_compaction_mandatory)
23378 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
23379 if (compact_reason >= 0)
23381 if (gc_heap_compact_reason_mandatory_p[compact_reason])
23382 is_compaction_mandatory = TRUE;
23385 #endif //GC_CONFIG_DRIVEN
23388 #ifdef GC_CONFIG_DRIVEN
23389 if (!is_compaction_mandatory)
23391 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
23392 // Note that we may want to change this to only checking every so often instead of every single GC.
23393 if (should_do_sweeping_gc (pol_max >= policy_compact))
23395 pol_max = policy_sweep;
23399 if (pol_max == policy_sweep)
23400 pol_max = policy_compact;
23403 #endif //GC_CONFIG_DRIVEN
23405 for (i = 0; i < n_heaps; i++)
23407 if (pol_max > g_heaps[i]->gc_policy)
23408 g_heaps[i]->gc_policy = pol_max;
23409 //get the segment while we are serialized
23410 if (g_heaps[i]->gc_policy == policy_expand)
23412 g_heaps[i]->new_heap_segment =
23413 g_heaps[i]->soh_get_segment_to_expand();
23414 if (!g_heaps[i]->new_heap_segment)
23416 set_expand_in_full_gc (condemned_gen_number);
23417 //we are out of memory, cancel the expansion
23418 g_heaps[i]->gc_policy = policy_compact;
23423 BOOL is_full_compacting_gc = FALSE;
23425 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
23427 full_gc_counts[gc_type_compacting]++;
23428 is_full_compacting_gc = TRUE;
23431 for (i = 0; i < n_heaps; i++)
23433 //copy the card and brick tables
23434 if (g_gc_card_table!= g_heaps[i]->card_table)
23436 g_heaps[i]->copy_brick_card_table();
23439 if (is_full_compacting_gc)
23441 g_heaps[i]->loh_alloc_since_cg = 0;
23446 dprintf(3, ("Starting all gc threads after compaction decision"));
23447 gc_t_join.restart();
23450 should_compact = (gc_policy >= policy_compact);
23451 should_expand = (gc_policy >= policy_expand);
23453 #else //MULTIPLE_HEAPS
23455 //safe place to delete large heap segments
23456 if (condemned_gen_number == max_generation)
23458 rearrange_uoh_segments ();
23461 if (maxgen_size_inc_p && provisional_mode_triggered)
23463 pm_trigger_full_gc = true;
23464 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
23468 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
23469 if (settings.demotion)
23470 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
23472 #ifdef GC_CONFIG_DRIVEN
23473 BOOL is_compaction_mandatory = FALSE;
23474 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
23475 if (compact_reason >= 0)
23476 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
23478 if (!is_compaction_mandatory)
23480 if (should_do_sweeping_gc (should_compact))
23481 should_compact = FALSE;
23483 should_compact = TRUE;
23485 #endif //GC_CONFIG_DRIVEN
23487 if (should_compact && (condemned_gen_number == max_generation))
23489 full_gc_counts[gc_type_compacting]++;
23490 loh_alloc_since_cg = 0;
23493 #endif //MULTIPLE_HEAPS
23495 if (!pm_trigger_full_gc && pm_stress_on && provisional_mode_triggered)
23497 if ((settings.condemned_generation == (max_generation - 1)) &&
23498 ((settings.gc_index % 5) == 0))
23500 pm_trigger_full_gc = true;
23504 if (settings.condemned_generation == (max_generation - 1))
23506 if (provisional_mode_triggered)
23510 should_expand = FALSE;
23511 dprintf (GTC_LOG, ("h%d in PM cannot expand", heap_number));
23515 if (pm_trigger_full_gc)
23517 should_compact = FALSE;
23518 dprintf (GTC_LOG, ("h%d PM doing sweeping", heap_number));
23522 if (should_compact)
23524 dprintf (2,( "**** Doing Compacting GC ****"));
23528 #ifndef MULTIPLE_HEAPS
23529 heap_segment* new_heap_segment = soh_get_segment_to_expand();
23530 #endif //!MULTIPLE_HEAPS
23531 if (new_heap_segment)
23533 consing_gen = expand_heap(condemned_gen_number,
23538 // If we couldn't get a new segment, or we were able to
23539 // reserve one but no space to commit, we couldn't
23541 if (ephemeral_heap_segment != new_heap_segment)
23543 set_expand_in_full_gc (condemned_gen_number);
23544 should_expand = FALSE;
23547 generation_allocation_limit (condemned_gen1) =
23548 generation_allocation_pointer (condemned_gen1);
23549 if ((condemned_gen_number < max_generation))
23551 generation_allocator (older_gen)->commit_alloc_list_changes();
23553 // Fix the allocation area of the older generation
23554 fix_older_allocation_area (older_gen);
23556 assert (generation_allocation_segment (consing_gen) ==
23557 ephemeral_heap_segment);
23559 GCToEEInterface::DiagWalkSurvivors(__this, true);
23561 relocate_phase (condemned_gen_number, first_condemned_address);
23562 compact_phase (condemned_gen_number, first_condemned_address,
23563 (!settings.demotion && settings.promotion));
23564 fix_generation_bounds (condemned_gen_number, consing_gen);
23565 assert (generation_allocation_limit (youngest_generation) ==
23566 generation_allocation_pointer (youngest_generation));
23567 if (condemned_gen_number >= (max_generation -1))
23569 #ifdef MULTIPLE_HEAPS
23570 // this needs be serialized just because we have one
23571 // segment_standby_list/seg_table for all heaps. We should make it at least
23572 // so that when hoarding is not on we don't need this join because
23573 // decommitting memory can take a long time.
23574 //must serialize on deleting segments
23575 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
23576 if (gc_t_join.joined())
23578 for (int i = 0; i < n_heaps; i++)
23580 g_heaps[i]->rearrange_heap_segments(TRUE);
23582 gc_t_join.restart();
23585 rearrange_heap_segments(TRUE);
23586 #endif //MULTIPLE_HEAPS
23590 //fix the start_segment for the ephemeral generations
23591 for (int i = 0; i < max_generation; i++)
23593 generation* gen = generation_of (i);
23594 generation_start_segment (gen) = ephemeral_heap_segment;
23595 generation_allocation_segment (gen) = ephemeral_heap_segment;
23601 #ifdef FEATURE_PREMORTEM_FINALIZATION
23602 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
23603 (!settings.demotion && settings.promotion));
23604 #endif // FEATURE_PREMORTEM_FINALIZATION
23606 #ifdef MULTIPLE_HEAPS
23607 dprintf(3, ("Joining after end of compaction"));
23608 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
23609 if (gc_t_join.joined())
23611 //join all threads to make sure they are synchronized
23612 dprintf(3, ("Restarting after Promotion granted"));
23613 gc_t_join.restart();
23615 #endif //MULTIPLE_HEAPS
23618 sc.thread_number = heap_number;
23619 sc.promotion = FALSE;
23620 sc.concurrent = FALSE;
23621 // new generations bounds are set can call this guy
23622 if (settings.promotion && !settings.demotion)
23624 dprintf (2, ("Promoting EE roots for gen %d",
23625 condemned_gen_number));
23626 GCScan::GcPromotionsGranted(condemned_gen_number,
23627 max_generation, &sc);
23629 else if (settings.demotion)
23631 dprintf (2, ("Demoting EE roots for gen %d",
23632 condemned_gen_number));
23633 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
23638 gen0_big_free_spaces = 0;
23640 reset_pinned_queue_bos();
23641 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
23642 generation* gen = generation_of (gen_number);
23643 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
23644 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
23646 while (!pinned_plug_que_empty_p())
23648 mark* m = pinned_plug_of (deque_pinned_plug());
23649 size_t len = pinned_len (m);
23650 uint8_t* arr = (pinned_plug (m) - len);
23651 dprintf(3,("free [%Ix %Ix[ pin",
23652 (size_t)arr, (size_t)arr + len));
23655 assert (len >= Align (min_obj_size));
23656 make_unused_array (arr, len);
23657 // fix fully contained bricks + first one
23658 // if the array goes beyond the first brick
23659 size_t start_brick = brick_of (arr);
23660 size_t end_brick = brick_of (arr + len);
23661 if (end_brick != start_brick)
23664 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
23665 start_brick, end_brick, (size_t)arr));
23666 set_brick (start_brick,
23667 arr - brick_address (start_brick));
23668 size_t brick = start_brick+1;
23669 while (brick < end_brick)
23671 set_brick (brick, start_brick - brick);
23676 //when we take an old segment to make the new
23677 //ephemeral segment. we can have a bunch of
23678 //pinned plugs out of order going to the new ephemeral seg
23679 //and then the next plugs go back to max_generation
23680 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
23681 (heap_segment_reserved (ephemeral_heap_segment) > arr))
23684 while ((low <= arr) && (high > arr))
23687 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
23688 settings.demotion || !settings.promotion);
23689 dprintf (3, ("new free list generation %d", gen_number));
23691 gen = generation_of (gen_number);
23692 if (gen_number >= 1)
23693 low = generation_allocation_start (generation_of (gen_number-1));
23700 dprintf (3, ("new free list generation %d", max_generation));
23701 gen_number = max_generation;
23702 gen = generation_of (gen_number);
23705 dprintf(3,("threading it into generation %d", gen_number));
23706 thread_gap (arr, len, gen);
23707 add_gen_free (gen_number, len);
23713 for (int x = 0; x <= max_generation; x++)
23715 assert (generation_allocation_start (generation_of (x)));
23719 if (!settings.demotion && settings.promotion)
23721 //clear card for generation 1. generation 0 is empty
23722 clear_card_for_addresses (
23723 generation_allocation_start (generation_of (1)),
23724 generation_allocation_start (generation_of (0)));
23726 if (settings.promotion && !settings.demotion)
23728 uint8_t* start = generation_allocation_start (youngest_generation);
23730 assert (heap_segment_allocated (ephemeral_heap_segment) ==
23731 (start + Align (size (start))));
23737 //force promotion for sweep
23738 settings.promotion = TRUE;
23739 settings.compaction = FALSE;
23742 sc.thread_number = heap_number;
23743 sc.promotion = FALSE;
23744 sc.concurrent = FALSE;
23746 dprintf (2, ("**** Doing Mark and Sweep GC****"));
23748 if ((condemned_gen_number < max_generation))
23750 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
23751 generation_free_list_space (older_gen) = r_free_list_space;
23752 generation_free_obj_space (older_gen) = r_free_obj_space;
23753 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
23754 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
23755 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
23756 generation_sweep_allocated (older_gen) += dd_survived_size (dynamic_data_of (condemned_gen_number));
23757 generation_allocation_limit (older_gen) = r_allocation_limit;
23758 generation_allocation_pointer (older_gen) = r_allocation_pointer;
23759 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
23760 generation_allocation_segment (older_gen) = r_allocation_segment;
23763 if ((condemned_gen_number < max_generation))
23765 // Fix the allocation area of the older generation
23766 fix_older_allocation_area (older_gen);
23769 GCToEEInterface::DiagWalkSurvivors(__this, false);
23771 gen0_big_free_spaces = 0;
23772 make_free_lists (condemned_gen_number);
23773 recover_saved_pinned_info();
23775 #ifdef FEATURE_PREMORTEM_FINALIZATION
23776 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
23777 #endif // FEATURE_PREMORTEM_FINALIZATION
23778 // MTHTS: leave single thread for HT processing on plan_phase
23779 #ifdef MULTIPLE_HEAPS
23780 dprintf(3, ("Joining after end of sweep"));
23781 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
23782 if (gc_t_join.joined())
23783 #endif //MULTIPLE_HEAPS
23785 GCScan::GcPromotionsGranted(condemned_gen_number,
23786 max_generation, &sc);
23787 if (condemned_gen_number >= (max_generation -1))
23789 #ifdef MULTIPLE_HEAPS
23790 for (int i = 0; i < n_heaps; i++)
23792 g_heaps[i]->rearrange_heap_segments(FALSE);
23795 rearrange_heap_segments(FALSE);
23796 #endif //MULTIPLE_HEAPS
23799 #ifdef MULTIPLE_HEAPS
23800 //join all threads to make sure they are synchronized
23801 dprintf(3, ("Restarting after Promotion granted"));
23802 gc_t_join.restart();
23803 #endif //MULTIPLE_HEAPS
23807 for (int x = 0; x <= max_generation; x++)
23809 assert (generation_allocation_start (generation_of (x)));
23813 //clear card for generation 1. generation 0 is empty
23814 clear_card_for_addresses (
23815 generation_allocation_start (generation_of (1)),
23816 generation_allocation_start (generation_of (0)));
23817 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
23818 (generation_allocation_start (youngest_generation) +
23819 Align (min_obj_size))));
23822 //verify_partial();
23825 #pragma warning(pop)
23829 /*****************************
23830 Called after compact phase to fix all generation gaps
23831 ********************************/
23832 void gc_heap::fix_generation_bounds (int condemned_gen_number,
23833 generation* consing_gen)
23836 UNREFERENCED_PARAMETER(consing_gen);
23839 assert (generation_allocation_segment (consing_gen) ==
23840 ephemeral_heap_segment);
23842 //assign the planned allocation start to the generation
23843 int gen_number = condemned_gen_number;
23844 int bottom_gen = 0;
23846 while (gen_number >= bottom_gen)
23848 generation* gen = generation_of (gen_number);
23849 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
23850 if ((gen_number < max_generation) && ephemeral_promotion)
23852 make_unused_array (saved_ephemeral_plan_start[gen_number],
23853 saved_ephemeral_plan_start_size[gen_number]);
23855 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
23856 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
23857 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
23860 #ifdef MULTIPLE_HEAPS
23861 if (ephemeral_promotion)
23863 //we are creating a generation fault. set the cards.
23864 // and we are only doing this for multiple heaps because in the single heap scenario the
23865 // new ephemeral generations will be empty and there'll be no need to set cards for the
23866 // old ephemeral generations that got promoted into max_generation.
23867 ptrdiff_t delta = 0;
23868 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
23870 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
23871 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
23872 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
23873 while (card != end_card)
23879 #endif //MULTIPLE_HEAPS
23881 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
23882 //reset the allocated size
23884 uint8_t* start = generation_allocation_start (youngest_generation);
23885 if (settings.promotion && !settings.demotion)
23887 assert ((start + Align (size (start))) ==
23888 heap_segment_plan_allocated(ephemeral_heap_segment));
23891 heap_segment_allocated(ephemeral_heap_segment)=
23892 heap_segment_plan_allocated(ephemeral_heap_segment);
23896 uint8_t* gc_heap::generation_limit (int gen_number)
23898 if (settings.promotion)
23900 if (gen_number <= 1)
23901 return heap_segment_reserved (ephemeral_heap_segment);
23903 return generation_allocation_start (generation_of ((gen_number - 2)));
23907 if (gen_number <= 0)
23908 return heap_segment_reserved (ephemeral_heap_segment);
23910 return generation_allocation_start (generation_of ((gen_number - 1)));
23914 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
23916 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23917 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
23918 assert ((start + size) <=
23919 heap_segment_reserved (ephemeral_heap_segment));
23920 if ((start + size) >
23921 heap_segment_committed (ephemeral_heap_segment))
23923 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
23931 uint8_t* gc_heap::allocate_at_end (size_t size)
23933 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23934 size = Align (size);
23935 uint8_t* result = start;
23936 // only called to allocate a min obj so can't overflow here.
23937 assert ((start + size) <=
23938 heap_segment_reserved (ephemeral_heap_segment));
23939 //ensure_gap_allocation took care of it
23940 assert ((start + size) <=
23941 heap_segment_committed (ephemeral_heap_segment));
23942 heap_segment_allocated (ephemeral_heap_segment) += size;
23947 void gc_heap::make_free_lists (int condemned_gen_number)
23949 //Promotion has to happen in sweep case.
23950 assert (settings.promotion);
23952 generation* condemned_gen = generation_of (condemned_gen_number);
23953 uint8_t* start_address = generation_allocation_start (condemned_gen);
23955 size_t current_brick = brick_of (start_address);
23956 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23958 PREFIX_ASSUME(current_heap_segment != NULL);
23960 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23961 size_t end_brick = brick_of (end_address-1);
23962 make_free_args args;
23963 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23964 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23966 (generation_limit (args.free_list_gen_number)));
23967 args.free_list_gen = generation_of (args.free_list_gen_number);
23968 args.highest_plug = 0;
23970 if ((start_address < end_address) ||
23971 (condemned_gen_number == max_generation))
23975 if ((current_brick > end_brick))
23977 if (args.current_gen_limit == MAX_PTR)
23979 //We had an empty segment
23980 //need to allocate the generation start
23981 generation* gen = generation_of (max_generation);
23983 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23985 PREFIX_ASSUME(start_seg != NULL);
23987 uint8_t* gap = heap_segment_mem (start_seg);
23989 generation_allocation_start (gen) = gap;
23990 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23991 make_unused_array (gap, Align (min_obj_size));
23992 reset_allocation_pointers (gen, gap);
23993 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23994 max_generation, (size_t)gap));
23995 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23997 if (heap_segment_next_rw (current_heap_segment))
23999 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24000 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24001 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24011 int brick_entry = brick_table [ current_brick ];
24012 if ((brick_entry >= 0))
24014 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
24015 dprintf(3,("Fixing brick entry %Ix to %Ix",
24016 current_brick, (size_t)args.highest_plug));
24017 set_brick (current_brick,
24018 (args.highest_plug - brick_address (current_brick)));
24022 if ((brick_entry > -32768))
24026 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
24027 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
24029 assert ((brick_entry == -1));
24032 //init to -1 for faster find_first_object
24033 set_brick (current_brick, -1);
24041 int bottom_gen = 0;
24042 args.free_list_gen_number--;
24043 while (args.free_list_gen_number >= bottom_gen)
24046 generation* gen2 = generation_of (args.free_list_gen_number);
24047 gap = allocate_at_end (Align(min_obj_size));
24048 generation_allocation_start (gen2) = gap;
24049 reset_allocation_pointers (gen2, gap);
24050 dprintf(3,("Fixing generation start of %d to: %Ix",
24051 args.free_list_gen_number, (size_t)gap));
24052 PREFIX_ASSUME(gap != NULL);
24053 make_unused_array (gap, Align (min_obj_size));
24055 args.free_list_gen_number--;
24058 //reset the allocated size
24059 uint8_t* start2 = generation_allocation_start (youngest_generation);
24060 alloc_allocated = start2 + Align (size (start2));
24064 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
24066 assert ((tree != NULL));
24068 int right_node = node_right_child (tree);
24069 int left_node = node_left_child (tree);
24070 args->highest_plug = 0;
24073 if (! (0 == left_node))
24075 make_free_list_in_brick (tree + left_node, args);
24079 uint8_t* plug = tree;
24080 size_t gap_size = node_gap_size (tree);
24081 uint8_t* gap = (plug - gap_size);
24082 dprintf (3,("Making free list %Ix len %d in %d",
24083 //dprintf (3,("F: %Ix len %Ix in %d",
24084 (size_t)gap, gap_size, args->free_list_gen_number));
24085 args->highest_plug = tree;
24087 if (is_plug_padded (plug))
24089 dprintf (3, ("%Ix padded", plug));
24090 clear_plug_padded (plug);
24092 #endif //SHORT_PLUGS
24095 if ((args->current_gen_limit == MAX_PTR) ||
24096 ((plug >= args->current_gen_limit) &&
24097 ephemeral_pointer_p (plug)))
24099 dprintf(3,(" Crossing Generation boundary at %Ix",
24100 (size_t)args->current_gen_limit));
24101 if (!(args->current_gen_limit == MAX_PTR))
24103 args->free_list_gen_number--;
24104 args->free_list_gen = generation_of (args->free_list_gen_number);
24106 dprintf(3,( " Fixing generation start of %d to: %Ix",
24107 args->free_list_gen_number, (size_t)gap));
24109 reset_allocation_pointers (args->free_list_gen, gap);
24110 args->current_gen_limit = generation_limit (args->free_list_gen_number);
24112 if ((gap_size >= (2*Align (min_obj_size))))
24114 dprintf(3,(" Splitting the gap in two %Id left",
24116 make_unused_array (gap, Align(min_obj_size));
24117 gap_size = (gap_size - Align(min_obj_size));
24118 gap = (gap + Align(min_obj_size));
24122 make_unused_array (gap, gap_size);
24129 thread_gap (gap, gap_size, args->free_list_gen);
24130 add_gen_free (args->free_list_gen->gen_num, gap_size);
24132 if (! (0 == right_node))
24134 make_free_list_in_brick (tree + right_node, args);
24140 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
24142 assert (generation_allocation_start (gen));
24145 if ((gen->gen_num == 0) && (size > CLR_SIZE))
24147 gen0_big_free_spaces += size;
24150 assert ((heap_segment_rw (generation_start_segment (gen))!=
24151 ephemeral_heap_segment) ||
24152 (gap_start > generation_allocation_start (gen)));
24153 // The beginning of a segment gap is not aligned
24154 assert (size >= Align (min_obj_size));
24155 make_unused_array (gap_start, size,
24156 (!settings.concurrent && (gen != youngest_generation)),
24157 (gen->gen_num == max_generation));
24158 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
24160 if ((size >= min_free_list))
24162 generation_free_list_space (gen) += size;
24163 generation_allocator (gen)->thread_item (gap_start, size);
24167 generation_free_obj_space (gen) += size;
24172 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
24174 assert (generation_allocation_start (gen));
24175 if (size >= min_free_list)
24177 generation_free_list_space (gen) += size;
24178 generation_allocator (gen)->thread_item_front (gap_start, size);
24182 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
24184 dprintf (3, ("Making unused array [%Ix, %Ix[",
24185 (size_t)x, (size_t)(x+size)));
24186 assert (size >= Align (min_obj_size));
24188 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
24189 // check_batch_mark_array_bits (x, x+size);
24190 //#endif //VERIFY_HEAP && BACKGROUND_GC
24194 #ifdef BGC_SERVO_TUNING
24195 // Don't do this for servo tuning because it makes it even harder to regulate WS.
24196 if (!(bgc_tuning::enable_fl_tuning && bgc_tuning::fl_tuning_triggered))
24197 #endif //BGC_SERVO_TUNING
24199 reset_memory (x, size);
24202 ((CObjectHeader*)x)->SetFree(size);
24207 #error "This won't work on big endian platforms"
24210 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
24212 if (size_as_object < size)
24215 // If the size is more than 4GB, we need to create multiple objects because of
24216 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
24217 // size is ignored in regular object size computation.
24219 uint8_t * tmp = x + size_as_object;
24220 size_t remaining_size = size - size_as_object;
24222 while (remaining_size > UINT32_MAX)
24224 // Make sure that there will be at least Align(min_obj_size) left
24225 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
24226 - Align (min_obj_size, get_alignment_constant (FALSE));
24228 ((CObjectHeader*)tmp)->SetFree(current_size);
24230 remaining_size -= current_size;
24231 tmp += current_size;
24234 ((CObjectHeader*)tmp)->SetFree(remaining_size);
24239 clear_card_for_addresses (x, x + Align(size));
24242 // Clear memory set by make_unused_array.
24243 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
24245 // Also clear the sync block
24246 *(((PTR_PTR)x)-1) = 0;
24248 ((CObjectHeader*)x)->UnsetFree();
24253 #error "This won't work on big endian platforms"
24256 // The memory could have been cleared in the meantime. We have to mirror the algorithm
24257 // from make_unused_array since we cannot depend on the object sizes in memory.
24258 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
24260 if (size_as_object < size)
24262 uint8_t * tmp = x + size_as_object;
24263 size_t remaining_size = size - size_as_object;
24265 while (remaining_size > UINT32_MAX)
24267 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
24268 - Align (min_obj_size, get_alignment_constant (FALSE));
24270 ((CObjectHeader*)tmp)->UnsetFree();
24272 remaining_size -= current_size;
24273 tmp += current_size;
24276 ((CObjectHeader*)tmp)->UnsetFree();
24279 UNREFERENCED_PARAMETER(size);
24284 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
24286 uint8_t* candidate = 0;
24290 if (tree < old_address)
24292 if ((cn = node_right_child (tree)) != 0)
24294 assert (candidate < tree);
24297 Prefetch (tree - 8);
24303 else if (tree > old_address)
24305 if ((cn = node_left_child (tree)) != 0)
24308 Prefetch (tree - 8);
24316 if (tree <= old_address)
24318 else if (candidate)
24324 #ifdef FEATURE_BASICFREEZE
24325 bool gc_heap::frozen_object_p (Object* obj)
24327 heap_segment* seg = seg_mapping_table_segment_of ((uint8_t*)obj);
24328 return heap_segment_read_only_p (seg);
24330 #endif // FEATURE_BASICFREEZE
24332 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
24334 uint8_t* old_address = *pold_address;
24335 if (!((old_address >= gc_low) && (old_address < gc_high)))
24336 #ifdef MULTIPLE_HEAPS
24338 UNREFERENCED_PARAMETER(thread);
24339 if (old_address == 0)
24341 gc_heap* hp = heap_of (old_address);
24342 if ((hp == this) ||
24343 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
24346 #else //MULTIPLE_HEAPS
24348 #endif //MULTIPLE_HEAPS
24349 // delta translates old_address into address_gc (old_address);
24350 size_t brick = brick_of (old_address);
24351 int brick_entry = brick_table [ brick ];
24352 uint8_t* new_address = old_address;
24353 if (! ((brick_entry == 0)))
24357 while (brick_entry < 0)
24359 brick = (brick + brick_entry);
24360 brick_entry = brick_table [ brick ];
24362 uint8_t* old_loc = old_address;
24364 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
24366 if ((node <= old_loc))
24367 new_address = (old_address + node_relocation_distance (node));
24370 if (node_left_p (node))
24372 dprintf(3,(" L: %Ix", (size_t)node));
24373 new_address = (old_address +
24374 (node_relocation_distance (node) +
24375 node_gap_size (node)));
24380 brick_entry = brick_table [ brick ];
24386 *pold_address = new_address;
24390 #ifdef FEATURE_LOH_COMPACTION
24391 if (settings.loh_compaction)
24393 heap_segment* pSegment = seg_mapping_table_segment_of ((uint8_t*)old_address);
24394 #ifdef MULTIPLE_HEAPS
24395 if (heap_segment_heap (pSegment)->loh_compacted_p)
24397 if (loh_compacted_p)
24400 size_t flags = pSegment->flags;
24401 if ((flags & heap_segment_flags_loh)
24402 #ifdef FEATURE_BASICFREEZE
24403 && !(flags & heap_segment_flags_readonly)
24407 *pold_address = old_address + loh_node_relocation_distance (old_address);
24411 #endif //FEATURE_LOH_COMPACTION
24415 gc_heap::check_class_object_demotion (uint8_t* obj)
24417 #ifdef COLLECTIBLE_CLASS
24418 if (is_collectible(obj))
24420 check_class_object_demotion_internal (obj);
24423 UNREFERENCED_PARAMETER(obj);
24424 #endif //COLLECTIBLE_CLASS
24427 #ifdef COLLECTIBLE_CLASS
24429 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
24431 if (settings.demotion)
24433 #ifdef MULTIPLE_HEAPS
24434 // We set the card without checking the demotion range 'cause at this point
24435 // the handle that points to the loader allocator object may or may not have
24436 // been relocated by other GC threads.
24437 set_card (card_of (obj));
24440 uint8_t* class_obj = get_class_object (obj);
24441 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
24442 uint8_t* temp_class_obj = class_obj;
24443 uint8_t** temp = &temp_class_obj;
24444 relocate_address (temp THREAD_NUMBER_ARG);
24446 check_demotion_helper (temp, obj);
24447 #endif //MULTIPLE_HEAPS
24451 #endif //COLLECTIBLE_CLASS
24454 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
24456 // detect if we are demoting an object
24457 if ((*pval < demotion_high) &&
24458 (*pval >= demotion_low))
24460 dprintf(3, ("setting card %Ix:%Ix",
24461 card_of((uint8_t*)pval),
24464 set_card (card_of (parent_obj));
24466 #ifdef MULTIPLE_HEAPS
24467 else if (settings.demotion)
24469 dprintf (4, ("Demotion active, computing heap_of object"));
24470 gc_heap* hp = heap_of (*pval);
24471 if ((*pval < hp->demotion_high) &&
24472 (*pval >= hp->demotion_low))
24474 dprintf(3, ("setting card %Ix:%Ix",
24475 card_of((uint8_t*)pval),
24478 set_card (card_of (parent_obj));
24481 #endif //MULTIPLE_HEAPS
24485 gc_heap::reloc_survivor_helper (uint8_t** pval)
24488 relocate_address (pval THREAD_NUMBER_ARG);
24490 check_demotion_helper (pval, (uint8_t*)pval);
24494 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
24497 if (contain_pointers (x))
24499 dprintf (3, ("$%Ix$", (size_t)x));
24501 go_through_object_nostart (method_table(x), x, s, pval,
24503 uint8_t* child = *pval;
24504 reloc_survivor_helper (pval);
24507 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
24512 check_class_object_demotion (x);
24516 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
24520 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
24521 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
24522 if (address_to_reloc)
24524 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
24527 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
24528 uint8_t* relocated_addr = *address_to_reloc;
24529 if ((relocated_addr < demotion_high) &&
24530 (relocated_addr >= demotion_low))
24532 dprintf (3, ("set card for location %Ix(%Ix)",
24533 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24535 set_card (card_of ((uint8_t*)address_to_set_card));
24537 #ifdef MULTIPLE_HEAPS
24538 else if (settings.demotion)
24540 gc_heap* hp = heap_of (relocated_addr);
24541 if ((relocated_addr < hp->demotion_high) &&
24542 (relocated_addr >= hp->demotion_low))
24544 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
24545 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24547 set_card (card_of ((uint8_t*)address_to_set_card));
24550 #endif //MULTIPLE_HEAPS
24553 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
24556 uint8_t* plug = pinned_plug (pinned_plug_entry);
24557 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
24558 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
24559 // address. Consider this scenario:
24560 // gen1 start | 3-ptr sized NP | PP
24562 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
24563 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
24564 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
24565 pre_plug_start += sizeof (uint8_t*);
24566 uint8_t** old_address = &pre_plug_start;
24568 uint8_t* old_val = (old_address ? *old_address : 0);
24569 relocate_address (old_address THREAD_NUMBER_ARG);
24572 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
24573 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
24576 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
24580 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
24583 uint8_t* plug = pinned_plug (pinned_plug_entry);
24587 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
24588 //if ((x + s) < plug)
24590 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
24591 // x, (x + s), (plug- (x + s)), plug));
24592 // GCToOSInterface::DebugBreak();
24595 relocate_pre_plug_info (pinned_plug_entry);
24598 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
24600 uint8_t* saved_plug_info_start = 0;
24601 uint8_t** saved_info_to_relocate = 0;
24605 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
24606 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24610 saved_plug_info_start = (plug - sizeof (plug_and_gap));
24611 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24614 uint8_t** current_saved_info_to_relocate = 0;
24615 uint8_t* child = 0;
24617 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
24619 if (contain_pointers (x))
24621 dprintf (3,("$%Ix$", (size_t)x));
24623 go_through_object_nostart (method_table(x), x, s, pval,
24625 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
24627 if ((uint8_t*)pval >= end)
24629 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
24630 child = *current_saved_info_to_relocate;
24631 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
24632 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
24633 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
24637 reloc_survivor_helper (pval);
24642 check_class_object_demotion (x);
24645 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
24648 while (x < plug_end)
24650 size_t s = size (x);
24651 uint8_t* next_obj = x + Align (s);
24652 Prefetch (next_obj);
24653 relocate_obj_helper (x, s);
24659 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
24660 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
24662 #if defined (_DEBUG) && defined (VERIFY_HEAP)
24663 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
24665 if (!verify_pinned_queue_p)
24668 if (settings.heap_expansion)
24671 for (size_t i = 0; i < mark_stack_tos; i++)
24673 mark& m = mark_stack_array[i];
24675 mark* pinned_plug_entry = pinned_plug_of(i);
24677 if (pinned_plug_entry->has_post_plug_info() &&
24678 pinned_plug_entry->post_short_p() &&
24679 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
24681 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
24682 // object after pin
24683 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
24684 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
24685 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
24687 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
24689 if (node_gap_size (next_obj) != *post_plug_debug)
24691 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
24692 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
24696 // can't do node_relocation_distance here as it clears the left bit.
24697 //if (node_relocation_distance (next_obj) != *post_plug_debug)
24698 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
24700 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
24701 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
24704 if (node_left_child (next_obj) > 0)
24706 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
24712 dprintf (3, ("%s verified", msg));
24715 UNREFERENCED_PARAMETER(msg);
24716 #endif // _DEBUG && VERIFY_HEAP
24719 #ifdef COLLECTIBLE_CLASS
24720 // We don't want to burn another ptr size space for pinned plugs to record this so just
24721 // set the card unconditionally for collectible objects if we are demoting.
24723 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
24725 if (settings.demotion)
24727 set_card (card_of (obj));
24730 #endif //COLLECTIBLE_CLASS
24732 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
24735 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
24736 BOOL is_pinned = (plug == p_plug);
24737 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
24739 plug_end += sizeof (gap_reloc_pair);
24741 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
24742 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
24744 verify_pins_with_post_plug_info("begin reloc short surv");
24746 while (x < plug_end)
24748 if (check_short_obj_p && ((DWORD)(plug_end - x) < (DWORD)min_pre_pin_obj_size))
24750 dprintf (3, ("last obj %Ix is short", x));
24754 #ifdef COLLECTIBLE_CLASS
24755 if (pinned_plug_entry->post_short_collectible_p())
24756 unconditional_set_card_collectible (x);
24757 #endif //COLLECTIBLE_CLASS
24759 // Relocate the saved references based on bits set.
24760 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
24761 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24762 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24764 if (pinned_plug_entry->post_short_bit_p (i))
24766 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24772 #ifdef COLLECTIBLE_CLASS
24773 if (pinned_plug_entry->pre_short_collectible_p())
24774 unconditional_set_card_collectible (x);
24775 #endif //COLLECTIBLE_CLASS
24777 relocate_pre_plug_info (pinned_plug_entry);
24779 // Relocate the saved references based on bits set.
24780 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
24781 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24782 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24784 if (pinned_plug_entry->pre_short_bit_p (i))
24786 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24794 size_t s = size (x);
24795 uint8_t* next_obj = x + Align (s);
24796 Prefetch (next_obj);
24798 if (next_obj >= plug_end)
24800 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
24801 next_obj, plug, plug_end));
24803 verify_pins_with_post_plug_info("before reloc short obj");
24805 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
24809 relocate_obj_helper (x, s);
24816 verify_pins_with_post_plug_info("end reloc short surv");
24819 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
24820 BOOL check_last_object_p,
24821 mark* pinned_plug_entry)
24823 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24824 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24826 if (check_last_object_p)
24828 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
24832 relocate_survivor_helper (plug, plug_end);
24836 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
24838 assert ((tree != NULL));
24840 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
24841 tree, args->last_plug,
24842 (tree + node_left_child (tree)),
24843 (tree + node_right_child (tree)),
24844 node_gap_size (tree)));
24846 if (node_left_child (tree))
24848 relocate_survivors_in_brick (tree + node_left_child (tree), args);
24851 uint8_t* plug = tree;
24852 BOOL has_post_plug_info_p = FALSE;
24853 BOOL has_pre_plug_info_p = FALSE;
24855 if (tree == oldest_pinned_plug)
24857 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24858 &has_post_plug_info_p);
24859 assert (tree == pinned_plug (args->pinned_plug_entry));
24861 dprintf (3, ("tree is the oldest pin: %Ix", tree));
24863 if (args->last_plug)
24865 size_t gap_size = node_gap_size (tree);
24866 uint8_t* gap = (plug - gap_size);
24867 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
24868 assert (gap_size >= Align (min_obj_size));
24869 uint8_t* last_plug_end = gap;
24871 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24874 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
24879 assert (!has_pre_plug_info_p);
24882 args->last_plug = plug;
24883 args->is_shortened = has_post_plug_info_p;
24884 if (has_post_plug_info_p)
24886 dprintf (3, ("setting %Ix as shortened", plug));
24888 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
24890 if (node_right_child (tree))
24892 relocate_survivors_in_brick (tree + node_right_child (tree), args);
24897 void gc_heap::update_oldest_pinned_plug()
24899 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
24902 void gc_heap::relocate_survivors (int condemned_gen_number,
24903 uint8_t* first_condemned_address)
24905 generation* condemned_gen = generation_of (condemned_gen_number);
24906 uint8_t* start_address = first_condemned_address;
24907 size_t current_brick = brick_of (start_address);
24908 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24910 PREFIX_ASSUME(current_heap_segment != NULL);
24912 uint8_t* end_address = 0;
24914 reset_pinned_queue_bos();
24915 update_oldest_pinned_plug();
24917 end_address = heap_segment_allocated (current_heap_segment);
24919 size_t end_brick = brick_of (end_address - 1);
24920 relocate_args args;
24921 args.is_shortened = FALSE;
24922 args.pinned_plug_entry = 0;
24923 args.last_plug = 0;
24926 if (current_brick > end_brick)
24928 if (args.last_plug)
24931 assert (!(args.is_shortened));
24932 relocate_survivors_in_plug (args.last_plug,
24933 heap_segment_allocated (current_heap_segment),
24935 args.pinned_plug_entry);
24938 args.last_plug = 0;
24941 if (heap_segment_next_rw (current_heap_segment))
24943 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24944 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24945 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24954 int brick_entry = brick_table [ current_brick ];
24956 if (brick_entry >= 0)
24958 relocate_survivors_in_brick (brick_address (current_brick) +
24967 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24969 if (check_last_object_p)
24971 size += sizeof (gap_reloc_pair);
24972 mark* entry = args->pinned_plug_entry;
24974 if (args->is_shortened)
24976 assert (entry->has_post_plug_info());
24977 entry->swap_post_plug_and_saved_for_profiler();
24981 assert (entry->has_pre_plug_info());
24982 entry->swap_pre_plug_and_saved_for_profiler();
24986 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24987 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24988 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24990 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24992 if (check_last_object_p)
24994 mark* entry = args->pinned_plug_entry;
24996 if (args->is_shortened)
24998 entry->swap_post_plug_and_saved_for_profiler();
25002 entry->swap_pre_plug_and_saved_for_profiler();
25007 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
25009 assert ((tree != NULL));
25010 if (node_left_child (tree))
25012 walk_relocation_in_brick (tree + node_left_child (tree), args);
25015 uint8_t* plug = tree;
25016 BOOL has_pre_plug_info_p = FALSE;
25017 BOOL has_post_plug_info_p = FALSE;
25019 if (tree == oldest_pinned_plug)
25021 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
25022 &has_post_plug_info_p);
25023 assert (tree == pinned_plug (args->pinned_plug_entry));
25026 if (args->last_plug != 0)
25028 size_t gap_size = node_gap_size (tree);
25029 uint8_t* gap = (plug - gap_size);
25030 uint8_t* last_plug_end = gap;
25031 size_t last_plug_size = (last_plug_end - args->last_plug);
25032 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
25033 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
25035 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
25036 if (!check_last_object_p)
25038 assert (last_plug_size >= Align (min_obj_size));
25041 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
25045 assert (!has_pre_plug_info_p);
25048 dprintf (3, ("set args last plug to plug: %Ix", plug));
25049 args->last_plug = plug;
25050 args->is_shortened = has_post_plug_info_p;
25052 if (node_right_child (tree))
25054 walk_relocation_in_brick (tree + node_right_child (tree), args);
25058 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
25060 generation* condemned_gen = generation_of (settings.condemned_generation);
25061 uint8_t* start_address = generation_allocation_start (condemned_gen);
25062 size_t current_brick = brick_of (start_address);
25063 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
25065 PREFIX_ASSUME(current_heap_segment != NULL);
25067 reset_pinned_queue_bos();
25068 update_oldest_pinned_plug();
25069 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25070 walk_relocate_args args;
25071 args.is_shortened = FALSE;
25072 args.pinned_plug_entry = 0;
25073 args.last_plug = 0;
25074 args.profiling_context = profiling_context;
25079 if (current_brick > end_brick)
25081 if (args.last_plug)
25083 walk_plug (args.last_plug,
25084 (heap_segment_allocated (current_heap_segment) - args.last_plug),
25087 args.last_plug = 0;
25089 if (heap_segment_next_rw (current_heap_segment))
25091 current_heap_segment = heap_segment_next_rw (current_heap_segment);
25092 current_brick = brick_of (heap_segment_mem (current_heap_segment));
25093 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25102 int brick_entry = brick_table [ current_brick ];
25103 if (brick_entry >= 0)
25105 walk_relocation_in_brick (brick_address (current_brick) +
25114 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
25116 if (type == walk_for_gc)
25117 walk_survivors_relocation (context, fn);
25118 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
25119 else if (type == walk_for_bgc)
25120 walk_survivors_for_bgc (context, fn);
25121 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
25123 assert (!"unknown type!");
25126 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
25127 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
25129 assert(settings.concurrent);
25131 for (int i = max_generation; i < total_generation_count; i++)
25133 int align_const = get_alignment_constant (i == max_generation);
25134 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (i)));
25138 uint8_t* o = heap_segment_mem (seg);
25139 uint8_t* end = heap_segment_allocated (seg);
25143 if (method_table(o) == g_gc_pFreeObjectMethodTable)
25145 o += Align (size (o), align_const);
25149 // It's survived. Make a fake plug, starting at o,
25150 // and send the event
25152 uint8_t* plug_start = o;
25154 while (method_table(o) != g_gc_pFreeObjectMethodTable)
25156 o += Align (size (o), align_const);
25163 uint8_t* plug_end = o;
25167 0, // Reloc distance == 0 as this is non-compacting
25169 false, // Non-compacting
25173 seg = heap_segment_next (seg);
25177 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
25179 void gc_heap::relocate_phase (int condemned_gen_number,
25180 uint8_t* first_condemned_address)
25183 sc.thread_number = heap_number;
25184 sc.promotion = FALSE;
25185 sc.concurrent = FALSE;
25187 #ifdef MULTIPLE_HEAPS
25188 //join all threads to make sure they are synchronized
25189 dprintf(3, ("Joining after end of plan"));
25190 gc_t_join.join(this, gc_join_begin_relocate_phase);
25191 if (gc_t_join.joined())
25194 //join all threads to make sure they are synchronized
25195 dprintf(3, ("Restarting for relocation"));
25196 gc_t_join.restart();
25198 #endif //MULTIPLE_HEAPS
25200 dprintf (2,("---- Relocate phase -----"));
25202 dprintf(3,("Relocating roots"));
25203 GCScan::GcScanRoots(GCHeap::Relocate,
25204 condemned_gen_number, max_generation, &sc);
25206 verify_pins_with_post_plug_info("after reloc stack");
25208 #ifdef BACKGROUND_GC
25209 if (gc_heap::background_running_p())
25211 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
25213 #endif //BACKGROUND_GC
25215 #ifdef FEATURE_CARD_MARKING_STEALING
25216 // for card marking stealing, do the other relocations *before* we scan the older generations
25217 // this gives us a chance to make up for imbalance in these phases later
25219 dprintf(3, ("Relocating survivors"));
25220 relocate_survivors(condemned_gen_number,
25221 first_condemned_address);
25224 #ifdef FEATURE_PREMORTEM_FINALIZATION
25225 dprintf(3, ("Relocating finalization data"));
25226 finalize_queue->RelocateFinalizationData(condemned_gen_number,
25228 #endif // FEATURE_PREMORTEM_FINALIZATION
25232 dprintf(3, ("Relocating handle table"));
25233 GCScan::GcScanHandles(GCHeap::Relocate,
25234 condemned_gen_number, max_generation, &sc);
25236 #endif // FEATURE_CARD_MARKING_STEALING
25238 if (condemned_gen_number != max_generation)
25240 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
25241 if (!card_mark_done_soh)
25242 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
25244 dprintf (3, ("Relocating cross generation pointers on heap %d", heap_number));
25245 mark_through_cards_for_segments(&gc_heap::relocate_address, TRUE THIS_ARG);
25246 verify_pins_with_post_plug_info("after reloc cards");
25247 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
25248 card_mark_done_soh = true;
25249 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
25252 if (condemned_gen_number != max_generation)
25254 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
25255 if (!card_mark_done_uoh)
25256 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
25258 dprintf (3, ("Relocating cross generation pointers for uoh objects on heap %d", heap_number));
25259 for (int i = uoh_start_generation; i < total_generation_count; i++)
25261 #ifndef ALLOW_REFERENCES_IN_POH
25262 if (i != poh_generation)
25263 #endif //ALLOW_REFERENCES_IN_POH
25264 mark_through_cards_for_uoh_objects(&gc_heap::relocate_address, i, TRUE THIS_ARG);
25267 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
25268 card_mark_done_uoh = true;
25269 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
25274 #ifdef FEATURE_LOH_COMPACTION
25275 if (loh_compacted_p)
25277 assert (settings.condemned_generation == max_generation);
25278 relocate_in_loh_compact();
25281 #endif //FEATURE_LOH_COMPACTION
25283 relocate_in_uoh_objects (loh_generation);
25286 #ifdef ALLOW_REFERENCES_IN_POH
25287 relocate_in_uoh_objects (poh_generation);
25290 #ifndef FEATURE_CARD_MARKING_STEALING
25291 // moved this code *before* we scan the older generations via mark_through_cards_xxx
25292 // this gives us a chance to have mark_through_cards_xxx make up for imbalance in the other relocations
25294 dprintf(3,("Relocating survivors"));
25295 relocate_survivors (condemned_gen_number,
25296 first_condemned_address);
25299 #ifdef FEATURE_PREMORTEM_FINALIZATION
25300 dprintf(3,("Relocating finalization data"));
25301 finalize_queue->RelocateFinalizationData (condemned_gen_number,
25303 #endif // FEATURE_PREMORTEM_FINALIZATION
25308 dprintf(3,("Relocating handle table"));
25309 GCScan::GcScanHandles(GCHeap::Relocate,
25310 condemned_gen_number, max_generation, &sc);
25312 #endif // !FEATURE_CARD_MARKING_STEALING
25315 #if defined(MULTIPLE_HEAPS) && defined(FEATURE_CARD_MARKING_STEALING)
25316 if (condemned_gen_number != max_generation)
25318 // check the other heaps cyclically and try to help out where the relocation isn't done
25319 for (int i = 0; i < gc_heap::n_heaps; i++)
25321 int heap_number_to_look_at = (i + heap_number) % gc_heap::n_heaps;
25322 gc_heap* hp = gc_heap::g_heaps[heap_number_to_look_at];
25323 if (!hp->card_mark_done_soh)
25325 dprintf(3, ("Relocating cross generation pointers on heap %d", hp->heap_number));
25326 hp->mark_through_cards_for_segments(&gc_heap::relocate_address, TRUE THIS_ARG);
25327 hp->card_mark_done_soh = true;
25330 if (!hp->card_mark_done_uoh)
25332 dprintf(3, ("Relocating cross generation pointers for uoh objects on heap %d", hp->heap_number));
25333 for (int i = uoh_start_generation; i < total_generation_count; i++)
25335 #ifndef ALLOW_REFERENCES_IN_POH
25336 if (i != poh_generation)
25337 #endif //ALLOW_REFERENCES_IN_POH
25338 hp->mark_through_cards_for_uoh_objects(&gc_heap::relocate_address, i, TRUE THIS_ARG);
25340 hp->card_mark_done_uoh = true;
25344 #endif // MULTIPLE_HEAPS && FEATURE_CARD_MARKING_STEALING
25346 dprintf(2,( "---- End of Relocate phase ----"));
25349 // This compares to see if tree is the current pinned plug and returns info
25350 // for this pinned plug. Also advances the pinned queue if that's the case.
25352 // We don't change the values of the plug info if tree is not the same as
25353 // the current pinned plug - the caller is responsible for setting the right
25354 // values to begin with.
25356 // POPO TODO: We are keeping this temporarily as this is also used by realloc
25357 // where it passes FALSE to deque_p, change it to use the same optimization
25358 // as relocate. Not as essential since realloc is already a slow path.
25359 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
25360 BOOL* has_pre_plug_info_p,
25361 BOOL* has_post_plug_info_p,
25364 if (!pinned_plug_que_empty_p())
25366 mark* oldest_entry = oldest_pin();
25367 uint8_t* oldest_plug = pinned_plug (oldest_entry);
25368 if (tree == oldest_plug)
25370 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
25371 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
25375 deque_pinned_plug();
25378 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
25380 (*has_pre_plug_info_p ? 1 : 0),
25381 (*has_post_plug_info_p ? 1 : 0)));
25383 return oldest_entry;
25390 // This also deques the oldest entry and update the oldest plug
25391 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
25392 BOOL* has_post_plug_info_p)
25394 mark* oldest_entry = oldest_pin();
25395 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
25396 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
25398 deque_pinned_plug();
25399 update_oldest_pinned_plug();
25400 return oldest_entry;
25404 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25407 copy_cards_for_addresses (dest, src, len);
25409 clear_card_for_addresses (dest, dest + len);
25412 // POPO TODO: We should actually just recover the artificially made gaps here..because when we copy
25413 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
25414 // we won't need to individually recover each overwritten part of plugs.
25416 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25420 #ifdef BACKGROUND_GC
25421 if (current_c_gc_state == c_gc_state_marking)
25423 //TODO: should look to see whether we should consider changing this
25424 // to copy a consecutive region of the mark array instead.
25425 copy_mark_bits_for_addresses (dest, src, len);
25427 #endif //BACKGROUND_GC
25428 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25429 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25430 memcopy (dest - plug_skew, src - plug_skew, len);
25431 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25432 if (SoftwareWriteWatch::IsEnabledForGCHeap())
25434 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
25435 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
25436 // object at (src + len), so it can be ignored anyway.
25437 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
25439 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25440 copy_cards_range (dest, src, len, copy_cards_p);
25444 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
25447 uint8_t* reloc_plug = plug + args->last_plug_relocation;
25449 if (check_last_object_p)
25451 size += sizeof (gap_reloc_pair);
25452 mark* entry = args->pinned_plug_entry;
25454 if (args->is_shortened)
25456 assert (entry->has_post_plug_info());
25457 entry->swap_post_plug_and_saved();
25461 assert (entry->has_pre_plug_info());
25462 entry->swap_pre_plug_and_saved();
25466 int old_brick_entry = brick_table [brick_of (plug)];
25468 assert (node_relocation_distance (plug) == args->last_plug_relocation);
25470 #ifdef FEATURE_STRUCTALIGN
25471 ptrdiff_t alignpad = node_alignpad(plug);
25474 make_unused_array (reloc_plug - alignpad, alignpad);
25475 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
25477 // The alignment padding is straddling one or more bricks;
25478 // it has to be the last "object" of its first brick.
25479 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
25482 #else // FEATURE_STRUCTALIGN
25483 size_t unused_arr_size = 0;
25484 BOOL already_padded_p = FALSE;
25486 if (is_plug_padded (plug))
25488 already_padded_p = TRUE;
25489 clear_plug_padded (plug);
25490 unused_arr_size = Align (min_obj_size);
25492 #endif //SHORT_PLUGS
25493 if (node_realigned (plug))
25495 unused_arr_size += switch_alignment_size (already_padded_p);
25498 if (unused_arr_size != 0)
25500 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
25502 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
25504 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
25505 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
25506 // The alignment padding is straddling one or more bricks;
25507 // it has to be the last "object" of its first brick.
25508 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
25511 #endif // FEATURE_STRUCTALIGN
25514 if (is_plug_padded (plug))
25516 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
25518 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
25520 // The alignment padding is straddling one or more bricks;
25521 // it has to be the last "object" of its first brick.
25522 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
25525 #endif //SHORT_PLUGS
25527 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
25529 if (args->check_gennum_p)
25531 int src_gennum = args->src_gennum;
25532 if (src_gennum == -1)
25534 src_gennum = object_gennum (plug);
25537 int dest_gennum = object_gennum_plan (reloc_plug);
25539 if (src_gennum < dest_gennum)
25541 generation_allocation_size (generation_of (dest_gennum)) += size;
25545 size_t current_reloc_brick = args->current_compacted_brick;
25547 if (brick_of (reloc_plug) != current_reloc_brick)
25549 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
25550 current_reloc_brick, brick_of (reloc_plug)));
25552 if (args->before_last_plug)
25554 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
25555 current_reloc_brick,
25556 args->before_last_plug,
25557 (args->before_last_plug - brick_address (current_reloc_brick))));
25560 set_brick (current_reloc_brick,
25561 args->before_last_plug - brick_address (current_reloc_brick));
25564 current_reloc_brick = brick_of (reloc_plug);
25566 size_t end_brick = brick_of (reloc_plug + size-1);
25567 if (end_brick != current_reloc_brick)
25569 // The plug is straddling one or more bricks
25570 // It has to be the last plug of its first brick
25571 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
25572 current_reloc_brick, (size_t)reloc_plug,
25573 (reloc_plug - brick_address (current_reloc_brick))));
25576 set_brick (current_reloc_brick,
25577 reloc_plug - brick_address (current_reloc_brick));
25579 // update all intervening brick
25580 size_t brick = current_reloc_brick + 1;
25581 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
25582 brick, (end_brick - 1)));
25583 while (brick < end_brick)
25585 set_brick (brick, -1);
25588 // code last brick offset as a plug address
25589 args->before_last_plug = brick_address (end_brick) -1;
25590 current_reloc_brick = end_brick;
25591 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
25592 args->before_last_plug, current_reloc_brick));
25596 dprintf (3, ("still in the same brick: %Ix", end_brick));
25597 args->before_last_plug = reloc_plug;
25599 args->current_compacted_brick = current_reloc_brick;
25601 if (check_last_object_p)
25603 mark* entry = args->pinned_plug_entry;
25605 if (args->is_shortened)
25607 entry->swap_post_plug_and_saved();
25611 entry->swap_pre_plug_and_saved();
25616 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
25618 assert (tree != NULL);
25619 int left_node = node_left_child (tree);
25620 int right_node = node_right_child (tree);
25621 ptrdiff_t relocation = node_relocation_distance (tree);
25627 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
25628 compact_in_brick ((tree + left_node), args);
25631 uint8_t* plug = tree;
25632 BOOL has_pre_plug_info_p = FALSE;
25633 BOOL has_post_plug_info_p = FALSE;
25635 if (tree == oldest_pinned_plug)
25637 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
25638 &has_post_plug_info_p);
25639 assert (tree == pinned_plug (args->pinned_plug_entry));
25642 if (args->last_plug != 0)
25644 size_t gap_size = node_gap_size (tree);
25645 uint8_t* gap = (plug - gap_size);
25646 uint8_t* last_plug_end = gap;
25647 size_t last_plug_size = (last_plug_end - args->last_plug);
25648 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
25649 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
25651 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
25652 if (!check_last_object_p)
25654 assert (last_plug_size >= Align (min_obj_size));
25657 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
25661 assert (!has_pre_plug_info_p);
25664 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
25665 args->last_plug = plug;
25666 args->last_plug_relocation = relocation;
25667 args->is_shortened = has_post_plug_info_p;
25671 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
25672 compact_in_brick ((tree + right_node), args);
25676 void gc_heap::recover_saved_pinned_info()
25678 reset_pinned_queue_bos();
25680 while (!(pinned_plug_que_empty_p()))
25682 mark* oldest_entry = oldest_pin();
25683 oldest_entry->recover_plug_info();
25684 #ifdef GC_CONFIG_DRIVEN
25685 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
25686 record_interesting_data_point (idp_pre_and_post_pin);
25687 else if (oldest_entry->has_pre_plug_info())
25688 record_interesting_data_point (idp_pre_pin);
25689 else if (oldest_entry->has_post_plug_info())
25690 record_interesting_data_point (idp_post_pin);
25691 #endif //GC_CONFIG_DRIVEN
25693 deque_pinned_plug();
25697 void gc_heap::compact_phase (int condemned_gen_number,
25698 uint8_t* first_condemned_address,
25701 #ifdef MULTIPLE_HEAPS
25702 dprintf(3, ("Joining after end of relocation"));
25703 gc_t_join.join(this, gc_join_relocate_phase_done);
25704 if (gc_t_join.joined())
25706 dprintf(3, ("Restarting for compaction"));
25707 gc_t_join.restart();
25709 #endif //MULTIPLE_HEAPS
25711 generation* condemned_gen = generation_of (condemned_gen_number);
25712 uint8_t* start_address = first_condemned_address;
25713 size_t current_brick = brick_of (start_address);
25714 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
25715 PREFIX_ASSUME(current_heap_segment != NULL);
25717 reset_pinned_queue_bos();
25718 update_oldest_pinned_plug();
25720 BOOL reused_seg = expand_reused_seg_p();
25723 for (int i = 1; i <= max_generation; i++)
25725 generation_allocation_size (generation_of (i)) = 0;
25729 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
25731 size_t end_brick = brick_of (end_address-1);
25733 args.last_plug = 0;
25734 args.before_last_plug = 0;
25735 args.current_compacted_brick = ~((size_t)1);
25736 args.is_shortened = FALSE;
25737 args.pinned_plug_entry = 0;
25738 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
25739 args.check_gennum_p = reused_seg;
25740 if (args.check_gennum_p)
25742 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25745 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
25746 first_condemned_address, brick_of (first_condemned_address)));
25748 #ifdef FEATURE_LOH_COMPACTION
25749 if (loh_compacted_p)
25753 #endif //FEATURE_LOH_COMPACTION
25755 if ((start_address < end_address) ||
25756 (condemned_gen_number == max_generation))
25760 if (current_brick > end_brick)
25762 if (args.last_plug != 0)
25764 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
25765 compact_plug (args.last_plug,
25766 (heap_segment_allocated (current_heap_segment) - args.last_plug),
25771 if (heap_segment_next_rw (current_heap_segment))
25773 current_heap_segment = heap_segment_next_rw (current_heap_segment);
25774 current_brick = brick_of (heap_segment_mem (current_heap_segment));
25775 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25776 args.last_plug = 0;
25777 if (args.check_gennum_p)
25779 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25785 if (args.before_last_plug !=0)
25787 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
25788 args.current_compacted_brick, (size_t)args.before_last_plug));
25789 assert (args.current_compacted_brick != ~1u);
25790 set_brick (args.current_compacted_brick,
25791 args.before_last_plug - brick_address (args.current_compacted_brick));
25797 int brick_entry = brick_table [ current_brick ];
25798 dprintf (3, ("B: %Ix(%Ix)->%Ix",
25799 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
25801 if (brick_entry >= 0)
25803 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
25812 recover_saved_pinned_info();
25814 concurrent_print_time_delta ("compact end");
25816 dprintf(2,("---- End of Compact phase ----"));
25819 #ifdef MULTIPLE_HEAPS
25822 #pragma warning(push)
25823 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25825 void gc_heap::gc_thread_stub (void* arg)
25827 gc_heap* heap = (gc_heap*)arg;
25828 if (!gc_thread_no_affinitize_p)
25830 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
25831 // CPU groups because the process mask, processor number, and group number are all
25832 // readily available.
25833 set_thread_affinity_for_heap (heap->heap_number, heap_select::find_proc_no_from_heap_no (heap->heap_number));
25836 // server GC threads run at a higher priority than normal.
25837 GCToOSInterface::BoostThreadPriority();
25838 _alloca (256*heap->heap_number);
25839 heap->gc_thread_function();
25842 #pragma warning(pop)
25845 #endif //MULTIPLE_HEAPS
25847 #ifdef BACKGROUND_GC
25850 #pragma warning(push)
25851 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25853 void gc_heap::bgc_thread_stub (void* arg)
25855 gc_heap* heap = (gc_heap*)arg;
25856 heap->bgc_thread = GCToEEInterface::GetThread();
25857 assert(heap->bgc_thread != nullptr);
25858 heap->bgc_thread_function();
25861 #pragma warning(pop)
25864 void gc_heap::background_drain_mark_list (int thread)
25866 #ifndef MULTIPLE_HEAPS
25867 UNREFERENCED_PARAMETER(thread);
25868 #endif //!MULTIPLE_HEAPS
25870 size_t saved_c_mark_list_index = c_mark_list_index;
25872 if (saved_c_mark_list_index)
25874 concurrent_print_time_delta ("SML");
25876 while (c_mark_list_index != 0)
25878 size_t current_index = c_mark_list_index - 1;
25879 uint8_t* o = c_mark_list [current_index];
25880 background_mark_object (o THREAD_NUMBER_ARG);
25881 c_mark_list_index--;
25883 if (saved_c_mark_list_index)
25885 concurrent_print_time_delta ("EML");
25888 fire_drain_mark_list_event (saved_c_mark_list_index);
25892 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
25893 #ifdef MULTIPLE_HEAPS
25894 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
25895 // them. So we can use the same static variables.
25896 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25898 // Whenever we call this method there may have been preceding object promotions. So set
25899 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25900 // based on the how the scanning proceeded).
25901 s_fUnscannedPromotions = TRUE;
25903 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
25904 // the state of this thread's portion of the dependent handle table. That's because promotions on other
25905 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
25906 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
25907 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
25908 // as all the others or they'll get out of step).
25911 // The various worker threads are all currently racing in this code. We need to work out if at least
25912 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
25913 // dependent handle table when both of the following conditions apply:
25914 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
25915 // object happens to correspond to a primary in one of our handles we might potentially have to
25916 // promote the associated secondary).
25917 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
25919 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
25920 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
25921 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
25922 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
25923 // follows below. Note that we can't read this outside of the join since on any iteration apart from
25924 // the first threads will be racing between reading this value and completing their previous
25925 // iteration's table scan.
25927 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
25928 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
25929 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
25930 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
25931 // we're safely joined.
25932 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25933 s_fUnpromotedHandles = TRUE;
25935 // Synchronize all the threads so we can read our state variables safely. The following shared
25936 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
25937 // single thread inside the join.
25938 bgc_t_join.join(this, gc_join_scan_dependent_handles);
25939 if (bgc_t_join.joined())
25941 // We're synchronized so it's safe to read our shared state variables. We update another shared
25942 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25943 // the loop. We scan if there has been at least one object promotion since last time and at least
25944 // one thread has a dependent handle table with a potential handle promotion possible.
25945 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25947 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25948 // value for the next call if we're terminating the loop).
25949 s_fUnscannedPromotions = FALSE;
25950 s_fUnpromotedHandles = FALSE;
25952 if (!s_fScanRequired)
25954 uint8_t* all_heaps_max = 0;
25955 uint8_t* all_heaps_min = MAX_PTR;
25957 for (i = 0; i < n_heaps; i++)
25959 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25960 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25961 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25962 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25964 for (i = 0; i < n_heaps; i++)
25966 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25967 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25971 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25972 bgc_t_join.restart();
25975 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25976 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25977 // global flag indicating that at least one object promotion may have occurred (the usual comment
25978 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25979 // exit the method since we unconditionally set this variable on method entry anyway).
25980 if (background_process_mark_overflow (sc->concurrent))
25981 s_fUnscannedPromotions = TRUE;
25983 // If we decided that no scan was required we can terminate the loop now.
25984 if (!s_fScanRequired)
25987 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25988 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25989 // could miss noting the promotion of some primary objects).
25990 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25991 if (bgc_t_join.joined())
25993 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25994 bgc_t_join.restart();
25997 // If the portion of the dependent handle table managed by this worker has handles that could still be
25998 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25999 // could require a rescan of handles on this or other workers.
26000 if (GCScan::GcDhUnpromotedHandlesExist(sc))
26001 if (GCScan::GcDhReScan(sc))
26002 s_fUnscannedPromotions = TRUE;
26006 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
26008 // Whenever we call this method there may have been preceding object promotions. So set
26009 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
26010 // based on the how the scanning proceeded).
26011 bool fUnscannedPromotions = true;
26013 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
26014 // scan without performing any new promotions.
26015 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
26017 // On each iteration of the loop start with the assumption that no further objects have been promoted.
26018 fUnscannedPromotions = false;
26020 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
26021 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
26022 // additional objects now appear to be promoted and we should set the flag.
26023 if (background_process_mark_overflow (sc->concurrent))
26024 fUnscannedPromotions = true;
26026 // Perform the scan and set the flag if any promotions resulted.
26027 if (GCScan::GcDhReScan (sc))
26028 fUnscannedPromotions = true;
26031 // Perform a last processing of any overflowed mark stack.
26032 background_process_mark_overflow (sc->concurrent);
26034 #endif //MULTIPLE_HEAPS
26036 void gc_heap::recover_bgc_settings()
26038 if ((settings.condemned_generation < max_generation) && gc_heap::background_running_p())
26040 dprintf (2, ("restoring bgc settings"));
26041 settings = saved_bgc_settings;
26042 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
26046 void gc_heap::allow_fgc()
26048 assert (bgc_thread == GCToEEInterface::GetThread());
26049 bool bToggleGC = false;
26051 if (g_fSuspensionPending > 0)
26053 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
26056 GCToEEInterface::DisablePreemptiveGC();
26061 BOOL gc_heap::should_commit_mark_array()
26063 return (gc_heap::background_running_p() || (current_bgc_state == bgc_initialized));
26066 void gc_heap::clear_commit_flag()
26068 for (int i = max_generation; i < total_generation_count; i++)
26070 generation* gen = generation_of (i);
26071 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26074 if (seg->flags & heap_segment_flags_ma_committed)
26076 seg->flags &= ~heap_segment_flags_ma_committed;
26079 if (seg->flags & heap_segment_flags_ma_pcommitted)
26081 seg->flags &= ~heap_segment_flags_ma_pcommitted;
26084 seg = heap_segment_next (seg);
26089 void gc_heap::clear_commit_flag_global()
26091 #ifdef MULTIPLE_HEAPS
26092 for (int i = 0; i < n_heaps; i++)
26094 g_heaps[i]->clear_commit_flag();
26097 clear_commit_flag();
26098 #endif //MULTIPLE_HEAPS
26101 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
26104 size_t markw = mark_word_of (begin);
26105 size_t markw_end = mark_word_of (end);
26107 while (markw < markw_end)
26109 if (mark_array_addr[markw])
26111 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
26112 markw, mark_array_addr[markw], mark_word_address (markw)));
26118 UNREFERENCED_PARAMETER(begin);
26119 UNREFERENCED_PARAMETER(end);
26120 UNREFERENCED_PARAMETER(mark_array_addr);
26124 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
26126 uint32_t* new_card_table,
26127 uint8_t* new_lowest_address)
26129 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26130 uint8_t* end = heap_segment_reserved (seg);
26132 uint8_t* lowest = hp->background_saved_lowest_address;
26133 uint8_t* highest = hp->background_saved_highest_address;
26135 uint8_t* commit_start = NULL;
26136 uint8_t* commit_end = NULL;
26137 size_t commit_flag = 0;
26139 if ((highest >= start) &&
26142 if ((start >= lowest) && (end <= highest))
26144 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
26145 start, end, lowest, highest));
26146 commit_flag = heap_segment_flags_ma_committed;
26150 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
26151 start, end, lowest, highest));
26152 commit_flag = heap_segment_flags_ma_pcommitted;
26155 commit_start = max (lowest, start);
26156 commit_end = min (highest, end);
26158 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
26163 if (new_card_table == 0)
26165 new_card_table = g_gc_card_table;
26168 if (hp->card_table != new_card_table)
26170 if (new_lowest_address == 0)
26172 new_lowest_address = g_gc_lowest_address;
26175 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
26176 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
26178 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
26179 hp->card_table, new_card_table,
26180 hp->mark_array, ma));
26182 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
26188 seg->flags |= commit_flag;
26194 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
26196 size_t beg_word = mark_word_of (begin);
26197 size_t end_word = mark_word_of (align_on_mark_word (end));
26198 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
26199 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
26200 size_t size = (size_t)(commit_end - commit_start);
26202 #ifdef SIMPLE_DPRINTF
26203 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
26205 beg_word, end_word,
26206 (end_word - beg_word) * sizeof (uint32_t),
26207 &mark_array_addr[beg_word],
26208 &mark_array_addr[end_word],
26209 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
26210 commit_start, commit_end,
26212 #endif //SIMPLE_DPRINTF
26214 if (virtual_commit (commit_start, size, gc_oh_num::none))
26216 // We can only verify the mark array is cleared from begin to end, the first and the last
26217 // page aren't necessarily all cleared 'cause they could be used by other segments or
26219 verify_mark_array_cleared (begin, end, mark_array_addr);
26224 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
26229 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
26231 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26232 uint8_t* end = heap_segment_reserved (seg);
26234 #ifdef MULTIPLE_HEAPS
26235 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
26236 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
26238 uint8_t* lowest = background_saved_lowest_address;
26239 uint8_t* highest = background_saved_highest_address;
26240 #endif //MULTIPLE_HEAPS
26242 if ((highest >= start) &&
26245 start = max (lowest, start);
26246 end = min (highest, end);
26247 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
26256 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
26258 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
26260 heap_segment_reserved (seg),
26262 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
26264 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
26267 BOOL gc_heap::commit_mark_array_bgc_init()
26269 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
26270 lowest_address, highest_address, mark_array));
26272 for (int i = max_generation; i < total_generation_count; i++)
26274 generation* gen = generation_of (i);
26275 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26278 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
26280 if (!(seg->flags & heap_segment_flags_ma_committed))
26282 // For ro segments they could always be only partially in range so we'd
26283 // be calling this at the beginning of every BGC. We are not making this
26284 // more efficient right now - ro segments are currently only used by redhawk.
26285 if (heap_segment_read_only_p (seg))
26287 if ((heap_segment_mem (seg) >= lowest_address) &&
26288 (heap_segment_reserved (seg) <= highest_address))
26290 if (commit_mark_array_by_seg (seg, mark_array))
26292 seg->flags |= heap_segment_flags_ma_committed;
26301 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
26302 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
26303 if (commit_mark_array_by_range (start, end, mark_array))
26305 seg->flags |= heap_segment_flags_ma_pcommitted;
26315 // For normal segments they are by design completely in range so just
26316 // commit the whole mark array for each seg.
26317 if (commit_mark_array_by_seg (seg, mark_array))
26319 if (seg->flags & heap_segment_flags_ma_pcommitted)
26321 seg->flags &= ~heap_segment_flags_ma_pcommitted;
26323 seg->flags |= heap_segment_flags_ma_committed;
26332 seg = heap_segment_next (seg);
26339 // This function doesn't check the commit flag since it's for a new array -
26340 // the mark_array flag for these segments will remain the same.
26341 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
26343 dprintf (GC_TABLE_LOG, ("committing existing segs on MA %Ix", new_mark_array_addr));
26345 for (int i = max_generation; i < total_generation_count; i++)
26347 generation* gen = generation_of (i);
26348 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26351 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
26356 seg = heap_segment_next (seg);
26359 #ifdef MULTIPLE_HEAPS
26360 if (new_heap_segment)
26362 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
26367 #endif //MULTIPLE_HEAPS
26373 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
26375 #ifdef MULTIPLE_HEAPS
26376 for (int i = 0; i < n_heaps; i++)
26378 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
26384 if (!commit_new_mark_array (new_mark_array))
26388 #endif //MULTIPLE_HEAPS
26393 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
26395 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
26396 // been set to NULL.
26397 if (mark_array == NULL)
26402 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
26404 size_t flags = seg->flags;
26406 if ((flags & heap_segment_flags_ma_committed) ||
26407 (flags & heap_segment_flags_ma_pcommitted))
26409 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26410 uint8_t* end = heap_segment_reserved (seg);
26412 if (flags & heap_segment_flags_ma_pcommitted)
26414 start = max (lowest_address, start);
26415 end = min (highest_address, end);
26418 size_t beg_word = mark_word_of (start);
26419 size_t end_word = mark_word_of (align_on_mark_word (end));
26420 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
26421 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
26422 size_t size = (size_t)(decommit_end - decommit_start);
26424 #ifdef SIMPLE_DPRINTF
26425 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
26427 beg_word, end_word,
26428 (end_word - beg_word) * sizeof (uint32_t),
26429 &mark_array[beg_word],
26430 &mark_array[end_word],
26431 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
26432 decommit_start, decommit_end,
26434 #endif //SIMPLE_DPRINTF
26436 if (decommit_start < decommit_end)
26438 if (!virtual_decommit (decommit_start, size, gc_oh_num::none))
26440 dprintf (GC_TABLE_LOG, ("decommit on %Ix for %Id bytes failed",
26441 decommit_start, size));
26442 assert (!"decommit failed");
26446 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
26450 void gc_heap::background_mark_phase ()
26452 verify_mark_array_cleared();
26455 sc.thread_number = heap_number;
26456 sc.promotion = TRUE;
26457 sc.concurrent = FALSE;
26460 BOOL cooperative_mode = TRUE;
26461 #ifndef MULTIPLE_HEAPS
26462 const int thread = heap_number;
26463 #endif //!MULTIPLE_HEAPS
26465 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
26467 assert (settings.concurrent);
26469 if (gen0_must_clear_bricks > 0)
26470 gen0_must_clear_bricks--;
26472 background_soh_alloc_count = 0;
26473 background_uoh_alloc_count = 0;
26474 bgc_overflow_count = 0;
26476 bpromoted_bytes (heap_number) = 0;
26477 static uint32_t num_sizedrefs = 0;
26479 background_min_overflow_address = MAX_PTR;
26480 background_max_overflow_address = 0;
26481 background_min_soh_overflow_address = MAX_PTR;
26482 background_max_soh_overflow_address = 0;
26483 processed_soh_overflow_p = FALSE;
26486 //set up the mark lists from g_mark_list
26487 assert (g_mark_list);
26488 mark_list = g_mark_list;
26489 //dont use the mark list for full gc
26490 //because multiple segments are more complex to handle and the list
26491 //is likely to overflow
26492 mark_list_end = &mark_list [0];
26493 mark_list_index = &mark_list [0];
26495 c_mark_list_index = 0;
26497 #ifndef MULTIPLE_HEAPS
26498 shigh = (uint8_t*) 0;
26500 #endif //MULTIPLE_HEAPS
26502 generation* gen = generation_of (max_generation);
26504 dprintf(3,("BGC: stack marking"));
26505 sc.concurrent = TRUE;
26507 GCScan::GcScanRoots(background_promote_callback,
26508 max_generation, max_generation,
26513 dprintf(3,("BGC: finalization marking"));
26514 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
26517 size_t total_soh_size = generation_sizes (generation_of (max_generation));
26518 size_t total_loh_size = generation_size (loh_generation);
26519 size_t total_poh_size = generation_size (poh_generation);
26520 bgc_begin_loh_size = total_loh_size;
26521 bgc_begin_poh_size = total_poh_size;
26522 bgc_loh_size_increased = 0;
26523 bgc_poh_size_increased = 0;
26525 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id, poh: %Id", heap_number, total_loh_size, total_soh_size, total_poh_size));
26528 //concurrent_print_time_delta ("copying stack roots");
26529 concurrent_print_time_delta ("CS");
26531 FIRE_EVENT(BGC1stNonConEnd);
26533 expanded_in_fgc = FALSE;
26534 saved_overflow_ephemeral_seg = 0;
26535 current_bgc_state = bgc_reset_ww;
26537 // we don't need a join here - just whichever thread that gets here
26538 // first can change the states and call restart_vm.
26539 // this is not true - we can't let the EE run when we are scanning stack.
26540 // since we now allow reset ww to run concurrently and have a join for it,
26541 // we can do restart ee on the 1st thread that got here. Make sure we handle the
26542 // sizedref handles correctly.
26543 #ifdef MULTIPLE_HEAPS
26544 bgc_t_join.join(this, gc_join_restart_ee);
26545 if (bgc_t_join.joined())
26546 #endif //MULTIPLE_HEAPS
26548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26549 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
26550 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
26551 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
26552 concurrent_print_time_delta ("CRWW begin");
26554 #ifdef MULTIPLE_HEAPS
26555 for (int i = 0; i < n_heaps; i++)
26557 g_heaps[i]->reset_write_watch (FALSE);
26560 reset_write_watch (FALSE);
26561 #endif //MULTIPLE_HEAPS
26563 concurrent_print_time_delta ("CRWW");
26564 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26566 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
26568 // this c_write is not really necessary because restart_vm
26569 // has an instruction that will flush the cpu cache (interlocked
26570 // or whatever) but we don't want to rely on that.
26571 dprintf (GTC_LOG, ("setting cm_in_progress"));
26572 c_write (cm_in_progress, TRUE);
26574 assert (dont_restart_ee_p);
26575 dont_restart_ee_p = FALSE;
26578 GCToOSInterface::YieldThread (0);
26579 #ifdef MULTIPLE_HEAPS
26580 dprintf(3, ("Starting all gc threads for gc"));
26581 bgc_t_join.restart();
26582 #endif //MULTIPLE_HEAPS
26585 #ifdef MULTIPLE_HEAPS
26586 bgc_t_join.join(this, gc_join_after_reset);
26587 if (bgc_t_join.joined())
26588 #endif //MULTIPLE_HEAPS
26590 disable_preemptive (true);
26592 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26593 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
26594 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
26595 // pages during the concurrent reset.
26598 concurrent_print_time_delta ("CRWW begin");
26600 #ifdef MULTIPLE_HEAPS
26601 for (int i = 0; i < n_heaps; i++)
26603 g_heaps[i]->reset_write_watch (TRUE);
26606 reset_write_watch (TRUE);
26607 #endif //MULTIPLE_HEAPS
26609 concurrent_print_time_delta ("CRWW");
26610 #endif //WRITE_WATCH
26612 #ifdef MULTIPLE_HEAPS
26613 for (int i = 0; i < n_heaps; i++)
26615 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
26618 revisit_written_pages (TRUE, TRUE);
26619 #endif //MULTIPLE_HEAPS
26621 concurrent_print_time_delta ("CRW");
26622 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26624 #ifdef MULTIPLE_HEAPS
26625 for (int i = 0; i < n_heaps; i++)
26627 g_heaps[i]->current_bgc_state = bgc_mark_handles;
26630 current_bgc_state = bgc_mark_handles;
26631 #endif //MULTIPLE_HEAPS
26633 current_c_gc_state = c_gc_state_marking;
26635 enable_preemptive ();
26637 #ifdef MULTIPLE_HEAPS
26638 dprintf(3, ("Joining BGC threads after resetting writewatch"));
26639 bgc_t_join.restart();
26640 #endif //MULTIPLE_HEAPS
26643 disable_preemptive (true);
26645 if (num_sizedrefs > 0)
26647 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
26649 enable_preemptive ();
26651 #ifdef MULTIPLE_HEAPS
26652 bgc_t_join.join(this, gc_join_scan_sizedref_done);
26653 if (bgc_t_join.joined())
26655 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
26656 bgc_t_join.restart();
26658 #endif //MULTIPLE_HEAPS
26660 disable_preemptive (true);
26663 dprintf (3,("BGC: handle table marking"));
26664 GCScan::GcScanHandles(background_promote,
26665 max_generation, max_generation,
26667 //concurrent_print_time_delta ("concurrent marking handle table");
26668 concurrent_print_time_delta ("CRH");
26670 current_bgc_state = bgc_mark_stack;
26671 dprintf (2,("concurrent draining mark list"));
26672 background_drain_mark_list (thread);
26673 //concurrent_print_time_delta ("concurrent marking stack roots");
26674 concurrent_print_time_delta ("CRS");
26676 dprintf (2,("concurrent revisiting dirtied pages"));
26678 // tuning has shown that there are advantages in doing this 2 times
26679 revisit_written_pages (TRUE);
26680 revisit_written_pages (TRUE);
26682 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
26683 concurrent_print_time_delta ("CRre");
26685 enable_preemptive ();
26687 #ifdef MULTIPLE_HEAPS
26688 bgc_t_join.join(this, gc_join_concurrent_overflow);
26689 if (bgc_t_join.joined())
26691 uint8_t* all_heaps_max = 0;
26692 uint8_t* all_heaps_min = MAX_PTR;
26694 for (i = 0; i < n_heaps; i++)
26696 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
26698 g_heaps[i]->background_max_overflow_address,
26699 g_heaps[i]->background_min_overflow_address));
26700 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
26701 all_heaps_max = g_heaps[i]->background_max_overflow_address;
26702 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
26703 all_heaps_min = g_heaps[i]->background_min_overflow_address;
26705 for (i = 0; i < n_heaps; i++)
26707 g_heaps[i]->background_max_overflow_address = all_heaps_max;
26708 g_heaps[i]->background_min_overflow_address = all_heaps_min;
26710 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
26711 bgc_t_join.restart();
26713 #endif //MULTIPLE_HEAPS
26715 disable_preemptive (true);
26717 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
26718 bgc_overflow_count = 0;
26719 background_process_mark_overflow (TRUE);
26720 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
26721 bgc_overflow_count = 0;
26722 //concurrent_print_time_delta ("concurrent processing mark overflow");
26723 concurrent_print_time_delta ("CRov");
26725 // Stop all threads, crawl all stacks and revisit changed pages.
26726 FIRE_EVENT(BGC1stConEnd);
26728 dprintf (2, ("Stopping the EE"));
26730 enable_preemptive ();
26732 #ifdef MULTIPLE_HEAPS
26733 bgc_t_join.join(this, gc_join_suspend_ee);
26734 if (bgc_t_join.joined())
26736 bgc_threads_sync_event.Reset();
26738 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
26739 bgc_t_join.restart();
26741 #endif //MULTIPLE_HEAPS
26743 if (heap_number == 0)
26745 enter_spin_lock (&gc_lock);
26747 suspended_start_time = GetHighPrecisionTimeStamp();
26750 bgc_threads_sync_event.Set();
26754 bgc_threads_sync_event.Wait(INFINITE, FALSE);
26755 dprintf (2, ("bgc_threads_sync_event is signalled"));
26758 assert (settings.concurrent);
26759 assert (settings.condemned_generation == max_generation);
26761 dprintf (2, ("clearing cm_in_progress"));
26762 c_write (cm_in_progress, FALSE);
26764 bgc_alloc_lock->check();
26766 current_bgc_state = bgc_final_marking;
26768 //concurrent_print_time_delta ("concurrent marking ended");
26769 concurrent_print_time_delta ("CR");
26771 FIRE_EVENT(BGC2ndNonConBegin);
26773 mark_absorb_new_alloc();
26775 // We need a join here 'cause find_object would complain if the gen0
26776 // bricks of another heap haven't been fixed up. So we need to make sure
26777 // that every heap's gen0 bricks are fixed up before we proceed.
26778 #ifdef MULTIPLE_HEAPS
26779 bgc_t_join.join(this, gc_join_after_absorb);
26780 if (bgc_t_join.joined())
26781 #endif //MULTIPLE_HEAPS
26783 #ifdef BGC_SERVO_TUNING
26784 bgc_tuning::record_bgc_sweep_start();
26785 #endif //BGC_SERVO_TUNING
26787 #ifdef MULTIPLE_HEAPS
26788 dprintf(3, ("Joining BGC threads after absorb"));
26789 bgc_t_join.restart();
26790 #endif //MULTIPLE_HEAPS
26793 // give VM a chance to do work
26794 GCToEEInterface::GcBeforeBGCSweepWork();
26796 //reset the flag, indicating that the EE no longer expect concurrent
26798 sc.concurrent = FALSE;
26800 total_soh_size = generation_sizes (generation_of (max_generation));
26801 total_loh_size = generation_size (loh_generation);
26802 total_poh_size = generation_size (poh_generation);
26804 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id, poh: %Id", heap_number, total_loh_size, total_soh_size, total_poh_size));
26806 dprintf (2, ("nonconcurrent marking stack roots"));
26807 GCScan::GcScanRoots(background_promote,
26808 max_generation, max_generation,
26810 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
26811 concurrent_print_time_delta ("NRS");
26813 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
26815 dprintf (2, ("nonconcurrent marking handle table"));
26816 GCScan::GcScanHandles(background_promote,
26817 max_generation, max_generation,
26819 //concurrent_print_time_delta ("nonconcurrent marking handle table");
26820 concurrent_print_time_delta ("NRH");
26822 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
26823 revisit_written_pages (FALSE);
26824 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
26825 concurrent_print_time_delta ("NRre LOH");
26827 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26828 #ifdef MULTIPLE_HEAPS
26829 bgc_t_join.join(this, gc_join_disable_software_write_watch);
26830 if (bgc_t_join.joined())
26831 #endif // MULTIPLE_HEAPS
26833 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
26834 // avoid further perf penalty after the runtime is restarted
26835 SoftwareWriteWatch::DisableForGCHeap();
26837 #ifdef MULTIPLE_HEAPS
26838 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
26839 bgc_t_join.restart();
26840 #endif // MULTIPLE_HEAPS
26842 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26844 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
26845 bgc_overflow_count = 0;
26847 // Dependent handles need to be scanned with a special algorithm (see the header comment on
26848 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
26849 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
26850 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
26851 // The call to background_scan_dependent_handles is what will cycle through more iterations if
26852 // required and will also perform processing of any mark stack overflow once the dependent handle
26853 // table has been fully promoted.
26854 dprintf (2, ("1st dependent handle scan and process mark overflow"));
26855 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
26856 background_scan_dependent_handles (&sc);
26857 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
26858 concurrent_print_time_delta ("NR 1st Hov");
26860 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
26861 bgc_overflow_count = 0;
26863 #ifdef MULTIPLE_HEAPS
26864 bgc_t_join.join(this, gc_join_null_dead_short_weak);
26865 if (bgc_t_join.joined())
26866 #endif //MULTIPLE_HEAPS
26868 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
26870 #ifdef MULTIPLE_HEAPS
26871 dprintf(3, ("Joining BGC threads for short weak handle scan"));
26872 bgc_t_join.restart();
26873 #endif //MULTIPLE_HEAPS
26876 // null out the target of short weakref that were not promoted.
26877 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
26879 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
26880 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
26884 #ifdef MULTIPLE_HEAPS
26885 bgc_t_join.join(this, gc_join_scan_finalization);
26886 if (bgc_t_join.joined())
26888 dprintf(3, ("Joining BGC threads for finalization"));
26889 bgc_t_join.restart();
26891 #endif //MULTIPLE_HEAPS
26893 dprintf(3,("Marking finalization data"));
26894 //concurrent_print_time_delta ("bgc joined to mark finalization");
26895 concurrent_print_time_delta ("NRj");
26897 // finalize_queue->EnterFinalizeLock();
26898 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
26899 // finalize_queue->LeaveFinalizeLock();
26901 concurrent_print_time_delta ("NRF");
26904 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26905 bgc_overflow_count = 0;
26907 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26908 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26910 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26911 background_scan_dependent_handles (&sc);
26912 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26913 concurrent_print_time_delta ("NR 2nd Hov");
26915 #ifdef MULTIPLE_HEAPS
26916 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26917 if (bgc_t_join.joined())
26919 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26920 bgc_t_join.restart();
26922 #endif //MULTIPLE_HEAPS
26924 // null out the target of long weakref that were not promoted.
26925 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26926 concurrent_print_time_delta ("NR GcWeakPtrScan");
26928 #ifdef MULTIPLE_HEAPS
26929 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26930 if (bgc_t_join.joined())
26931 #endif //MULTIPLE_HEAPS
26933 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26934 // scan for deleted entries in the syncblk cache
26935 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26936 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26937 #ifdef MULTIPLE_HEAPS
26938 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26939 bgc_t_join.restart();
26940 #endif //MULTIPLE_HEAPS
26943 gen0_bricks_cleared = FALSE;
26945 dprintf (2, ("end of bgc mark: loh: %d, poh: %d, soh: %d",
26946 generation_size (loh_generation),
26947 generation_size (poh_generation),
26948 generation_sizes (generation_of (max_generation))));
26950 for (int gen_idx = max_generation; gen_idx < total_generation_count; gen_idx++)
26952 generation* gen = generation_of (gen_idx);
26953 dynamic_data* dd = dynamic_data_of (gen_idx);
26954 dd_begin_data_size (dd) = generation_size (gen_idx) -
26955 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26956 Align (size (generation_allocation_start (gen)));
26957 dd_survived_size (dd) = 0;
26958 dd_pinned_survived_size (dd) = 0;
26959 dd_artificial_pinned_survived_size (dd) = 0;
26960 dd_added_pinned_size (dd) = 0;
26963 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26964 PREFIX_ASSUME(seg != NULL);
26968 seg->flags &= ~heap_segment_flags_swept;
26970 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26975 if (seg == ephemeral_heap_segment)
26977 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26981 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26984 dprintf (2, ("seg %Ix background allocated is %Ix",
26985 heap_segment_mem (seg),
26986 heap_segment_background_allocated (seg)));
26987 seg = heap_segment_next_rw (seg);
26990 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26991 // we can't let the user code consume the left over parts in these alloc contexts.
26992 repair_allocation_contexts (FALSE);
26994 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26995 generation_free_list_space (generation_of (max_generation)),
26996 generation_free_obj_space (generation_of (max_generation))));
26998 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
27002 gc_heap::suspend_EE ()
27004 dprintf (2, ("suspend_EE"));
27005 #ifdef MULTIPLE_HEAPS
27006 gc_heap* hp = gc_heap::g_heaps[0];
27007 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
27009 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
27010 #endif //MULTIPLE_HEAPS
27013 #ifdef MULTIPLE_HEAPS
27015 gc_heap::bgc_suspend_EE ()
27017 for (int i = 0; i < n_heaps; i++)
27019 gc_heap::g_heaps[i]->reset_gc_done();
27022 dprintf (2, ("bgc_suspend_EE"));
27023 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
27025 gc_started = FALSE;
27026 for (int i = 0; i < n_heaps; i++)
27028 gc_heap::g_heaps[i]->set_gc_done();
27033 gc_heap::bgc_suspend_EE ()
27037 dprintf (2, ("bgc_suspend_EE"));
27038 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
27039 gc_started = FALSE;
27042 #endif //MULTIPLE_HEAPS
27045 gc_heap::restart_EE ()
27047 dprintf (2, ("restart_EE"));
27048 #ifdef MULTIPLE_HEAPS
27049 GCToEEInterface::RestartEE(FALSE);
27051 GCToEEInterface::RestartEE(FALSE);
27052 #endif //MULTIPLE_HEAPS
27055 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
27059 uint8_t* end = ((seg == ephemeral_heap_segment) ?
27060 generation_allocation_start (generation_of (max_generation-1)) :
27061 heap_segment_allocated (seg));
27062 return align_lower_page (end);
27066 return heap_segment_allocated (seg);
27070 void gc_heap::revisit_written_page (uint8_t* page,
27073 uint8_t*& last_page,
27074 uint8_t*& last_object,
27075 BOOL large_objects_p,
27076 size_t& num_marked_objects)
27078 uint8_t* start_address = page;
27080 int align_const = get_alignment_constant (!large_objects_p);
27081 uint8_t* high_address = end;
27082 uint8_t* current_lowest_address = background_saved_lowest_address;
27083 uint8_t* current_highest_address = background_saved_highest_address;
27084 BOOL no_more_loop_p = FALSE;
27087 #ifndef MULTIPLE_HEAPS
27088 const int thread = heap_number;
27089 #endif //!MULTIPLE_HEAPS
27091 if (large_objects_p)
27097 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
27098 || (start_address <= last_object))
27104 o = find_first_object (start_address, last_object);
27105 // We can visit the same object again, but on a different page.
27106 assert (o >= last_object);
27110 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
27111 (size_t)page, (size_t)o,
27112 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
27114 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
27118 if (concurrent_p && large_objects_p)
27120 bgc_alloc_lock->bgc_mark_set (o);
27122 if (((CObjectHeader*)o)->IsFree())
27124 s = unused_array_size (o);
27136 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
27138 assert (Align (s) >= Align (min_obj_size));
27140 uint8_t* next_o = o + Align (s, align_const);
27142 if (next_o >= start_address)
27144 #ifdef MULTIPLE_HEAPS
27147 // We set last_object here for SVR BGC here because SVR BGC has more than
27148 // one GC thread. When we have more than one GC thread we would run into this
27149 // situation if we skipped unmarked objects:
27150 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
27152 // bgc thread 2 marks X and all its current children.
27153 // user thread comes along and dirties more (and later) pages in X.
27154 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
27155 // on them because it had already skipped X. We need to detect that this object is now
27156 // marked and mark the children on the dirtied pages.
27157 // In the future if we have less BGC threads than we have heaps we should add
27158 // the check to the number of BGC threads.
27161 #endif //MULTIPLE_HEAPS
27163 if (contain_pointers (o) &&
27164 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
27165 background_marked (o)))
27167 dprintf (3, ("going through %Ix", (size_t)o));
27168 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
27169 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
27171 no_more_loop_p = TRUE;
27174 uint8_t* oo = *poo;
27176 num_marked_objects++;
27177 background_mark_object (oo THREAD_NUMBER_ARG);
27182 ((CObjectHeader*)o)->IsFree() &&
27183 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
27185 // We need to not skip the object here because of this corner scenario:
27186 // A large object was being allocated during BGC mark so we first made it
27187 // into a free object, then cleared its memory. In this loop we would detect
27188 // that it's a free object which normally we would skip. But by the next time
27189 // we call GetWriteWatch we could still be on this object and the object had
27190 // been made into a valid object and some of its memory was changed. We need
27191 // to be sure to process those written pages so we can't skip the object just
27194 // Similarly, when using software write watch, don't advance last_object when
27195 // the current object is a free object that spans beyond the current page or
27196 // high_address. Software write watch acquires gc_lock before the concurrent
27197 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
27198 // happen at that point and allocate from this free region, so when
27199 // revisit_written_pages() continues, it cannot skip now-valid objects in this
27201 no_more_loop_p = TRUE;
27206 if (concurrent_p && large_objects_p)
27208 bgc_alloc_lock->bgc_mark_done ();
27210 if (no_more_loop_p)
27217 #ifdef MULTIPLE_HEAPS
27220 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
27223 #endif //MULTIPLE_HEAPS
27228 dprintf (3,("Last object: %Ix", (size_t)last_object));
27229 last_page = align_write_watch_lower_page (o);
27237 // When reset_only_p is TRUE, we should only reset pages that are in range
27238 // because we need to consider the segments or part of segments that were
27239 // allocated out of range all live.
27240 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
27242 if (concurrent_p && !reset_only_p)
27244 current_bgc_state = bgc_revisit_soh;
27247 size_t total_dirtied_pages = 0;
27248 size_t total_marked_objects = 0;
27250 bool reset_watch_state = !!concurrent_p;
27251 bool is_runtime_suspended = !concurrent_p;
27252 BOOL small_object_segments = TRUE;
27253 for (int i = max_generation; i < total_generation_count; i++)
27255 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (i)));
27256 PREFIX_ASSUME(seg != NULL);
27260 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
27261 //we need to truncate to the base of the page because
27262 //some newly allocated could exist beyond heap_segment_allocated
27263 //and if we reset the last page write watch status,
27264 // they wouldn't be guaranteed to be visited -> gc hole.
27265 uintptr_t bcount = array_size;
27266 uint8_t* last_page = 0;
27267 uint8_t* last_object = heap_segment_mem (seg);
27268 uint8_t* high_address = 0;
27270 BOOL skip_seg_p = FALSE;
27274 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
27275 (heap_segment_reserved (seg) <= background_saved_highest_address))
27277 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
27278 heap_segment_mem (seg), heap_segment_reserved (seg)));
27285 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
27289 base_address = max (base_address, background_saved_lowest_address);
27290 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
27293 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
27294 heap_segment_mem (seg), heap_segment_reserved (seg)));
27301 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
27302 high_address = min (high_address, background_saved_highest_address);
27306 high_address = high_page (seg, concurrent_p);
27309 if ((base_address < high_address) &&
27310 (bcount >= array_size))
27312 ptrdiff_t region_size = high_address - base_address;
27313 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
27315 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27316 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
27317 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
27318 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
27320 if (!is_runtime_suspended)
27322 enter_spin_lock(&gc_lock);
27324 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27326 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
27327 (void**)background_written_addresses,
27328 &bcount, is_runtime_suspended);
27330 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27331 if (!is_runtime_suspended)
27333 leave_spin_lock(&gc_lock);
27335 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27339 total_dirtied_pages += bcount;
27341 dprintf (3, ("Found %d pages [%Ix, %Ix[",
27342 bcount, (size_t)base_address, (size_t)high_address));
27347 for (unsigned i = 0; i < bcount; i++)
27349 uint8_t* page = (uint8_t*)background_written_addresses[i];
27350 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
27351 (size_t)page, (size_t)high_address));
27352 if (page < high_address)
27354 //search for marked objects in the page
27355 revisit_written_page (page, high_address, concurrent_p,
27356 last_page, last_object,
27357 !small_object_segments,
27358 total_marked_objects);
27362 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
27363 assert (!"page shouldn't have exceeded limit");
27368 if (bcount >= array_size){
27369 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
27370 bcount = array_size;
27380 seg = heap_segment_next_rw (seg);
27383 if (i < loh_generation)
27387 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
27388 fire_revisit_event (total_dirtied_pages, total_marked_objects, FALSE);
27389 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
27390 total_dirtied_pages = 0;
27391 total_marked_objects = 0;
27394 if (concurrent_p && !reset_only_p)
27396 current_bgc_state = bgc_revisit_uoh;
27399 small_object_segments = FALSE;
27400 dprintf (3, ("now revisiting large object segments"));
27406 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
27410 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
27411 fire_revisit_event (total_dirtied_pages, total_marked_objects, TRUE);
27417 void gc_heap::background_grow_c_mark_list()
27419 assert (c_mark_list_index >= c_mark_list_length);
27420 BOOL should_drain_p = FALSE;
27422 #ifndef MULTIPLE_HEAPS
27423 const int thread = heap_number;
27424 #endif //!MULTIPLE_HEAPS
27426 dprintf (2, ("stack copy buffer overflow"));
27427 uint8_t** new_c_mark_list = 0;
27430 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
27432 should_drain_p = TRUE;
27436 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
27437 if (new_c_mark_list == 0)
27439 should_drain_p = TRUE;
27443 if (should_drain_p)
27446 dprintf (2, ("No more memory for the stacks copy, draining.."));
27447 //drain the list by marking its elements
27448 background_drain_mark_list (thread);
27452 assert (new_c_mark_list);
27453 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
27454 c_mark_list_length = c_mark_list_length*2;
27455 delete c_mark_list;
27456 c_mark_list = new_c_mark_list;
27460 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
27463 UNREFERENCED_PARAMETER(sc);
27464 //in order to save space on the array, mark the object,
27465 //knowing that it will be visited later
27466 assert (settings.concurrent);
27468 THREAD_NUMBER_FROM_CONTEXT;
27469 #ifndef MULTIPLE_HEAPS
27470 const int thread = 0;
27471 #endif //!MULTIPLE_HEAPS
27473 uint8_t* o = (uint8_t*)*ppObject;
27480 gc_heap* hp = gc_heap::heap_of (o);
27482 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
27487 if (flags & GC_CALL_INTERIOR)
27489 o = hp->find_object (o);
27494 #ifdef FEATURE_CONSERVATIVE_GC
27495 // For conservative GC, a value on stack may point to middle of a free object.
27496 // In this case, we don't need to promote the pointer.
27497 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
27501 #endif //FEATURE_CONSERVATIVE_GC
27504 ((CObjectHeader*)o)->Validate();
27507 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
27508 if (o && (size (o) > loh_size_threshold))
27510 dprintf (3, ("Brc %Ix", (size_t)o));
27513 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
27515 hpt->background_grow_c_mark_list();
27517 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
27518 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
27520 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
27523 void gc_heap::mark_absorb_new_alloc()
27525 fix_allocation_contexts (FALSE);
27527 gen0_bricks_cleared = FALSE;
27529 clear_gen0_bricks();
27532 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
27534 BOOL success = FALSE;
27535 BOOL thread_created = FALSE;
27536 dprintf (2, ("Preparing gc thread"));
27537 gh->bgc_threads_timeout_cs.Enter();
27538 if (!(gh->bgc_thread_running))
27540 dprintf (2, ("GC thread not running"));
27541 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
27544 thread_created = TRUE;
27549 dprintf (3, ("GC thread already running"));
27552 gh->bgc_threads_timeout_cs.Leave();
27555 FIRE_EVENT(GCCreateConcurrentThread_V1);
27560 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
27562 assert (background_gc_done_event.IsValid());
27564 //dprintf (2, ("Creating BGC thread"));
27566 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
27567 return gh->bgc_thread_running;
27570 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
27573 dprintf (3, ("Creating concurrent GC thread for the first time"));
27574 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
27578 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
27582 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
27586 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
27591 #ifdef MULTIPLE_HEAPS
27592 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
27594 UNREFERENCED_PARAMETER(number_of_heaps);
27595 #endif //MULTIPLE_HEAPS
27603 if (background_gc_done_event.IsValid())
27605 background_gc_done_event.CloseEvent();
27607 if (bgc_threads_sync_event.IsValid())
27609 bgc_threads_sync_event.CloseEvent();
27611 if (ee_proceed_event.IsValid())
27613 ee_proceed_event.CloseEvent();
27615 if (bgc_start_event.IsValid())
27617 bgc_start_event.CloseEvent();
27624 BOOL gc_heap::create_bgc_thread_support()
27628 //needs to have room for enough smallest objects fitting on a page
27629 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
27635 make_c_mark_list (parr);
27640 int gc_heap::check_for_ephemeral_alloc()
27642 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
27646 #ifdef MULTIPLE_HEAPS
27647 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
27648 #endif //MULTIPLE_HEAPS
27650 for (int i = 0; i < max_generation; i++)
27652 #ifdef MULTIPLE_HEAPS
27653 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
27655 if (get_new_allocation (i) <= 0)
27656 #endif //MULTIPLE_HEAPS
27658 gen = max (gen, i);
27669 // Wait for gc to finish sequential part
27670 void gc_heap::wait_to_proceed()
27672 assert (background_gc_done_event.IsValid());
27673 assert (bgc_start_event.IsValid());
27675 user_thread_wait(&ee_proceed_event, FALSE);
27678 // Start a new concurrent gc
27679 void gc_heap::start_c_gc()
27681 assert (background_gc_done_event.IsValid());
27682 assert (bgc_start_event.IsValid());
27684 //Need to make sure that the gc thread is in the right place.
27685 background_gc_done_event.Wait(INFINITE, FALSE);
27686 background_gc_done_event.Reset();
27687 bgc_start_event.Set();
27690 void gc_heap::do_background_gc()
27692 dprintf (2, ("starting a BGC"));
27693 #ifdef MULTIPLE_HEAPS
27694 for (int i = 0; i < n_heaps; i++)
27696 g_heaps[i]->init_background_gc();
27699 init_background_gc();
27700 #endif //MULTIPLE_HEAPS
27702 #ifdef BGC_SERVO_TUNING
27703 bgc_tuning::record_bgc_start();
27704 #endif //BGC_SERVO_TUNING
27706 //start the background gc
27709 //wait until we get restarted by the BGC.
27713 void gc_heap::kill_gc_thread()
27715 //assert (settings.concurrent == FALSE);
27717 // We are doing a two-stage shutdown now.
27718 // In the first stage, we do minimum work, and call ExitProcess at the end.
27719 // In the secodn stage, we have the Loader lock and only one thread is
27720 // alive. Hence we do not need to kill gc thread.
27721 background_gc_done_event.CloseEvent();
27722 bgc_start_event.CloseEvent();
27723 bgc_threads_timeout_cs.Destroy();
27727 void gc_heap::bgc_thread_function()
27729 assert (background_gc_done_event.IsValid());
27730 assert (bgc_start_event.IsValid());
27732 dprintf (3, ("gc_thread thread starting..."));
27734 BOOL do_exit = FALSE;
27736 bool cooperative_mode = true;
27737 bgc_thread_id.SetToCurrentThread();
27738 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
27741 // Wait for work to do...
27742 dprintf (3, ("bgc thread: waiting..."));
27744 cooperative_mode = enable_preemptive ();
27745 //current_thread->m_fPreemptiveGCDisabled = 0;
27747 uint32_t result = bgc_start_event.Wait(
27749 #ifdef MULTIPLE_HEAPS
27753 #endif //MULTIPLE_HEAPS
27755 #ifdef MULTIPLE_HEAPS
27759 #endif //MULTIPLE_HEAPS
27762 dprintf (2, ("gc thread: finished waiting"));
27764 // not calling disable_preemptive here 'cause we
27765 // can't wait for GC complete here - RestartEE will be called
27766 // when we've done the init work.
27768 if (result == WAIT_TIMEOUT)
27770 // Should join the bgc threads and terminate all of them
27772 dprintf (1, ("GC thread timeout"));
27773 bgc_threads_timeout_cs.Enter();
27774 if (!keep_bgc_threads_p)
27776 dprintf (2, ("GC thread exiting"));
27777 bgc_thread_running = FALSE;
27779 bgc_thread_id.Clear();
27782 bgc_threads_timeout_cs.Leave();
27787 dprintf (3, ("GC thread needed, not exiting"));
27791 // if we signal the thread with no concurrent work to do -> exit
27792 if (!settings.concurrent)
27794 dprintf (3, ("no concurrent GC needed, exiting"));
27797 gc_background_running = TRUE;
27798 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
27799 generation_free_list_space (generation_of (max_generation)),
27800 generation_free_obj_space (generation_of (max_generation)),
27801 dd_fragmentation (dynamic_data_of (max_generation))));
27805 current_bgc_state = bgc_not_in_process;
27807 enable_preemptive ();
27808 #ifdef MULTIPLE_HEAPS
27809 bgc_t_join.join(this, gc_join_done);
27810 if (bgc_t_join.joined())
27811 #endif //MULTIPLE_HEAPS
27813 enter_spin_lock (&gc_lock);
27814 dprintf (SPINLOCK_LOG, ("bgc Egc"));
27816 bgc_start_event.Reset();
27818 #ifdef MULTIPLE_HEAPS
27819 for (int gen = max_generation; gen < total_generation_count; gen++)
27821 size_t desired_per_heap = 0;
27822 size_t total_desired = 0;
27825 for (int i = 0; i < n_heaps; i++)
27828 dd = hp->dynamic_data_of (gen);
27829 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
27830 if (temp_total_desired < total_desired)
27833 total_desired = (size_t)MAX_PTR;
27836 total_desired = temp_total_desired;
27839 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
27841 for (int i = 0; i < n_heaps; i++)
27843 hp = gc_heap::g_heaps[i];
27844 dd = hp->dynamic_data_of (gen);
27845 dd_desired_allocation (dd) = desired_per_heap;
27846 dd_gc_new_allocation (dd) = desired_per_heap;
27847 dd_new_allocation (dd) = desired_per_heap;
27850 #endif //MULTIPLE_HEAPS
27851 #ifdef MULTIPLE_HEAPS
27853 #endif //MULTIPLE_HEAPS
27855 c_write (settings.concurrent, FALSE);
27856 gc_background_running = FALSE;
27857 keep_bgc_threads_p = FALSE;
27858 background_gc_done_event.Set();
27860 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27861 leave_spin_lock (&gc_lock);
27862 #ifdef MULTIPLE_HEAPS
27863 dprintf(1, ("End of BGC - starting all BGC threads"));
27864 bgc_t_join.restart();
27865 #endif //MULTIPLE_HEAPS
27867 // We can't disable preempt here because there might've been a GC already
27868 // started and decided to do a BGC and waiting for a BGC thread to restart
27869 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27870 // to restart the VM so we deadlock.
27871 //gc_heap::disable_preemptive (true);
27874 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27876 dprintf (3, ("bgc_thread thread exiting"));
27880 #ifdef BGC_SERVO_TUNING
27881 bool gc_heap::bgc_tuning::stepping_trigger (uint32_t current_memory_load, size_t current_gen2_count)
27883 if (!bgc_tuning::enable_fl_tuning)
27888 bool stepping_trigger_p = false;
27889 if (use_stepping_trigger_p)
27891 dprintf (BGC_TUNING_LOG, ("current ml: %d, goal: %d",
27892 current_memory_load, memory_load_goal));
27893 // We don't go all the way up to mem goal because if we do we could end up with every
27894 // BGC being triggered by stepping all the way up to goal, and when we actually reach
27895 // goal we have no time to react 'cause the next BGC could already be over goal.
27896 if ((current_memory_load <= (memory_load_goal * 2 / 3)) ||
27897 ((memory_load_goal > current_memory_load) &&
27898 ((memory_load_goal - current_memory_load) > (stepping_interval * 3))))
27900 int memory_load_delta = (int)current_memory_load - (int)last_stepping_mem_load;
27901 if (memory_load_delta >= (int)stepping_interval)
27903 stepping_trigger_p = (current_gen2_count == last_stepping_bgc_count);
27904 if (stepping_trigger_p)
27906 current_gen2_count++;
27909 dprintf (BGC_TUNING_LOG, ("current ml: %d - %d = %d (>= %d), gen2 count: %d->%d, stepping trigger: %s ",
27910 current_memory_load, last_stepping_mem_load, memory_load_delta, stepping_interval,
27911 last_stepping_bgc_count, current_gen2_count,
27912 (stepping_trigger_p ? "yes" : "no")));
27913 last_stepping_mem_load = current_memory_load;
27914 last_stepping_bgc_count = current_gen2_count;
27919 use_stepping_trigger_p = false;
27923 return stepping_trigger_p;
27926 // Note that I am doing this per heap but as we are in this calculation other
27927 // heaps could increase their fl alloc. We are okay with that inaccurancy.
27928 bool gc_heap::bgc_tuning::should_trigger_bgc_loh()
27930 if (fl_tuning_triggered)
27932 #ifdef MULTIPLE_HEAPS
27933 gc_heap* hp = g_heaps[0];
27935 gc_heap* hp = pGenGCHeap;
27936 #endif //MULTIPLE_HEAPS
27938 if (!(gc_heap::background_running_p()))
27940 size_t current_alloc = get_total_servo_alloc (loh_generation);
27941 tuning_calculation* current_gen_calc = &gen_calc[loh_generation - max_generation];
27943 if (current_alloc < current_gen_calc->last_bgc_end_alloc)
27945 dprintf (BGC_TUNING_LOG, ("BTL: current alloc: %Id, last alloc: %Id?",
27946 current_alloc, current_gen_calc->last_bgc_end_alloc));
27949 bool trigger_p = ((current_alloc - current_gen_calc->last_bgc_end_alloc) >= current_gen_calc->alloc_to_trigger);
27950 dprintf (2, ("BTL3: LOH a %Id, la: %Id(%Id), %Id",
27951 current_alloc, current_gen_calc->last_bgc_end_alloc,
27952 (current_alloc - current_gen_calc->last_bgc_end_alloc),
27953 current_gen_calc->alloc_to_trigger));
27957 dprintf (BGC_TUNING_LOG, ("BTL3: LOH detected (%Id - %Id) >= %Id, TRIGGER",
27958 current_alloc, current_gen_calc->last_bgc_end_alloc, current_gen_calc->alloc_to_trigger));
27967 bool gc_heap::bgc_tuning::should_trigger_bgc()
27969 if (!bgc_tuning::enable_fl_tuning || gc_heap::background_running_p())
27974 if (settings.reason == reason_bgc_tuning_loh)
27976 // TODO: this should be an assert because if the reason was reason_bgc_tuning_loh,
27977 // we should have already set to condemn max_generation but I'm keeping it
27978 // for now in case we are reverting it for other reasons.
27979 bgc_tuning::next_bgc_p = true;
27980 dprintf (BGC_TUNING_LOG, ("BTL LOH triggered"));
27984 if (!bgc_tuning::next_bgc_p &&
27985 !fl_tuning_triggered &&
27986 (gc_heap::settings.entry_memory_load >= (memory_load_goal * 2 / 3)) &&
27987 (gc_heap::full_gc_counts[gc_type_background] >= 2))
27991 gen_calc[0].first_alloc_to_trigger = gc_heap::get_total_servo_alloc (max_generation);
27992 gen_calc[1].first_alloc_to_trigger = gc_heap::get_total_servo_alloc (loh_generation);
27993 dprintf (BGC_TUNING_LOG, ("BTL[GTC] mem high enough: %d(goal: %d), %Id BGCs done, g2a=%Id, g3a=%Id, trigger FL tuning!",
27994 gc_heap::settings.entry_memory_load, memory_load_goal,
27995 gc_heap::full_gc_counts[gc_type_background],
27996 gen_calc[0].first_alloc_to_trigger,
27997 gen_calc[1].first_alloc_to_trigger));
28000 if (bgc_tuning::next_bgc_p)
28002 dprintf (BGC_TUNING_LOG, ("BTL started FL tuning"));
28006 if (!fl_tuning_triggered)
28011 // If the tuning started, we need to check if we've exceeded the alloc.
28013 bgc_tuning::tuning_calculation* current_gen_calc = 0;
28016 current_gen_calc = &bgc_tuning::gen_calc[index];
28018 #ifdef MULTIPLE_HEAPS
28019 gc_heap* hp = g_heaps[0];
28021 gc_heap* hp = pGenGCHeap;
28022 #endif //MULTIPLE_HEAPS
28024 size_t current_gen1_index = dd_collection_count (hp->dynamic_data_of (max_generation - 1));
28025 size_t gen1_so_far = current_gen1_index - gen1_index_last_bgc_end;
28027 if (current_gen_calc->alloc_to_trigger > 0)
28029 // We are specifically checking for gen2 here. LOH is covered by should_trigger_bgc_loh.
28030 size_t current_alloc = get_total_servo_alloc (max_generation);
28031 if ((current_alloc - current_gen_calc->last_bgc_end_alloc) >= current_gen_calc->alloc_to_trigger)
28033 dprintf (BGC_TUNING_LOG, ("BTL2: SOH detected (%Id - %Id) >= %Id, TRIGGER",
28034 current_alloc, current_gen_calc->last_bgc_end_alloc, current_gen_calc->alloc_to_trigger));
28035 settings.reason = reason_bgc_tuning_soh;
28043 bool gc_heap::bgc_tuning::should_delay_alloc (int gen_number)
28045 if ((gen_number != max_generation) || !bgc_tuning::enable_fl_tuning)
28048 if (current_c_gc_state == c_gc_state_planning)
28051 #ifdef MULTIPLE_HEAPS
28052 for (; i < gc_heap::n_heaps; i++)
28054 gc_heap* hp = gc_heap::g_heaps[i];
28055 size_t current_fl_size = generation_free_list_space (hp->generation_of (max_generation));
28056 size_t last_bgc_fl_size = hp->bgc_maxgen_end_fl_size;
28059 size_t current_fl_size = generation_free_list_space (generation_of (max_generation));
28060 size_t last_bgc_fl_size = bgc_maxgen_end_fl_size;
28061 #endif //MULTIPLE_HEAPS
28063 if (last_bgc_fl_size)
28065 float current_flr = (float) current_fl_size / (float)last_bgc_fl_size;
28066 if (current_flr < 0.4)
28068 dprintf (BGC_TUNING_LOG, ("BTL%d h%d last fl %Id, curr fl %Id (%.3f) d1",
28069 gen_number, i, last_bgc_fl_size, current_fl_size, current_flr));
28079 void gc_heap::bgc_tuning::update_bgc_start (int gen_number, size_t num_gen1s_since_end)
28081 int tuning_data_index = gen_number - max_generation;
28082 tuning_calculation* current_gen_calc = &gen_calc[tuning_data_index];
28083 tuning_stats* current_gen_stats = &gen_stats[tuning_data_index];
28085 size_t total_generation_size = get_total_generation_size (gen_number);
28086 ptrdiff_t current_bgc_fl_size = get_total_generation_fl_size (gen_number);
28088 double physical_gen_flr = (double)current_bgc_fl_size * 100.0 / (double)total_generation_size;
28090 ptrdiff_t artificial_additional_fl = 0;
28092 if (fl_tuning_triggered)
28094 artificial_additional_fl = ((current_gen_calc->end_gen_size_goal > total_generation_size) ? (current_gen_calc->end_gen_size_goal - total_generation_size) : 0);
28095 total_generation_size += artificial_additional_fl;
28096 current_bgc_fl_size += artificial_additional_fl;
28099 current_gen_calc->current_bgc_start_flr = (double)current_bgc_fl_size * 100.0 / (double)total_generation_size;
28101 size_t current_alloc = get_total_servo_alloc (gen_number);
28102 dprintf (BGC_TUNING_LOG, ("BTL%d: st a: %Id, la: %Id",
28103 gen_number, current_alloc, current_gen_stats->last_alloc));
28104 current_gen_stats->last_alloc_end_to_start = current_alloc - current_gen_stats->last_alloc;
28105 current_gen_stats->last_alloc = current_alloc;
28107 current_gen_calc->actual_alloc_to_trigger = current_alloc - current_gen_calc->last_bgc_end_alloc;
28109 dprintf (BGC_TUNING_LOG, ("BTL%d: st: %Id g1s (%Id->%Id/gen1) since end, flr: %.3f(afl: %Id, %.3f)",
28110 gen_number, actual_num_gen1s_to_trigger,
28111 current_gen_stats->last_alloc_end_to_start,
28112 (num_gen1s_since_end ? (current_gen_stats->last_alloc_end_to_start / num_gen1s_since_end) : 0),
28113 current_gen_calc->current_bgc_start_flr, artificial_additional_fl, physical_gen_flr));
28116 void gc_heap::bgc_tuning::record_bgc_start()
28118 if (!bgc_tuning::enable_fl_tuning)
28121 uint64_t elapsed_time_so_far = GetHighPrecisionTimeStamp() - process_start_time;
28123 // Note that younger gen's collection count is always updated with older gen's collections.
28124 // So to calcuate the actual # of gen1 occurred we really should take the # of gen2s into
28125 // acccount (and deduct from gen1's collection count). But right now I am using it for stats.
28126 size_t current_gen1_index = get_current_gc_index (max_generation - 1);
28128 dprintf (BGC_TUNING_LOG, ("BTL: g2t[st][g1 %Id]: %0.3f minutes",
28129 current_gen1_index,
28130 (double)elapsed_time_so_far / (double)1000000 / (double)60));
28132 actual_num_gen1s_to_trigger = current_gen1_index - gen1_index_last_bgc_end;
28133 gen1_index_last_bgc_start = current_gen1_index;
28135 update_bgc_start (max_generation, actual_num_gen1s_to_trigger);
28136 update_bgc_start (loh_generation, actual_num_gen1s_to_trigger);
28139 double convert_range (double lower, double upper, double num, double percentage)
28141 double d = num - lower;
28146 d = min ((upper - lower), d);
28147 return (d * percentage);
28151 double calculate_gradual_d (double delta_double, double step)
28153 bool changed_sign = false;
28154 if (delta_double < 0.0)
28156 delta_double = -delta_double;
28157 changed_sign = true;
28160 double current_lower_limit = 0;
28161 double current_ratio = 1.0;
28162 // Given a step, we will gradually reduce the weight of the portion
28164 // We reduce by *0.6 each time so there will be 3 iterations:
28165 // 1->0.6->0.36 (next one would be 0.216 and terminate the loop)
28166 // This will produce a result that's between 0 and 0.098.
28167 while (current_ratio > 0.22)
28169 res += convert_range (current_lower_limit, (current_lower_limit + step), delta_double, current_ratio);
28170 current_lower_limit += step;
28171 current_ratio *= 0.6;
28180 void gc_heap::bgc_tuning::update_bgc_sweep_start (int gen_number, size_t num_gen1s_since_start)
28182 int tuning_data_index = gen_number - max_generation;
28183 tuning_calculation* current_gen_calc = &gen_calc[tuning_data_index];
28184 tuning_stats* current_gen_stats = &gen_stats[tuning_data_index];
28186 size_t total_generation_size = 0;
28187 ptrdiff_t current_bgc_fl_size = 0;
28189 total_generation_size = get_total_generation_size (gen_number);
28190 current_bgc_fl_size = get_total_generation_fl_size (gen_number);
28192 double physical_gen_flr = (double)current_bgc_fl_size * 100.0 / (double)total_generation_size;
28194 ptrdiff_t artificial_additional_fl = 0;
28195 if (fl_tuning_triggered)
28197 artificial_additional_fl = ((current_gen_calc->end_gen_size_goal > total_generation_size) ? (current_gen_calc->end_gen_size_goal - total_generation_size) : 0);
28198 total_generation_size += artificial_additional_fl;
28199 current_bgc_fl_size += artificial_additional_fl;
28202 current_gen_calc->current_bgc_sweep_flr = (double)current_bgc_fl_size * 100.0 / (double)total_generation_size;
28204 size_t current_alloc = get_total_servo_alloc (gen_number);
28205 dprintf (BGC_TUNING_LOG, ("BTL%d: sw a: %Id, la: %Id",
28206 gen_number, current_alloc, current_gen_stats->last_alloc));
28207 current_gen_stats->last_alloc_start_to_sweep = current_alloc - current_gen_stats->last_alloc;
28208 // We are resetting gen2 alloc at sweep start.
28209 current_gen_stats->last_alloc = 0;
28211 #ifdef SIMPLE_DPRINTF
28212 dprintf (BGC_TUNING_LOG, ("BTL%d: sflr: %.3f%%->%.3f%% (%Id->%Id, %Id->%Id) (%Id:%Id-%Id/gen1) since start (afl: %Id, %.3f)",
28214 current_gen_calc->last_bgc_flr, current_gen_calc->current_bgc_sweep_flr,
28215 current_gen_calc->last_bgc_size, total_generation_size,
28216 current_gen_stats->last_bgc_fl_size, current_bgc_fl_size,
28217 num_gen1s_since_start, current_gen_stats->last_alloc_start_to_sweep,
28218 (num_gen1s_since_start? (current_gen_stats->last_alloc_start_to_sweep / num_gen1s_since_start) : 0),
28219 artificial_additional_fl, physical_gen_flr));
28220 #endif //SIMPLE_DPRINTF
28223 void gc_heap::bgc_tuning::record_bgc_sweep_start()
28225 if (!bgc_tuning::enable_fl_tuning)
28228 size_t current_gen1_index = get_current_gc_index (max_generation - 1);
28229 size_t num_gen1s_since_start = current_gen1_index - gen1_index_last_bgc_start;
28230 gen1_index_last_bgc_sweep = current_gen1_index;
28232 uint64_t elapsed_time_so_far = GetHighPrecisionTimeStamp() - process_start_time;
28233 dprintf (BGC_TUNING_LOG, ("BTL: g2t[sw][g1 %Id]: %0.3f minutes",
28234 current_gen1_index,
28235 (double)elapsed_time_so_far / (double)1000000 / (double)60));
28237 update_bgc_sweep_start (max_generation, num_gen1s_since_start);
28238 update_bgc_sweep_start (loh_generation, num_gen1s_since_start);
28241 void gc_heap::bgc_tuning::calculate_tuning (int gen_number, bool use_this_loop_p)
28243 BOOL use_kd_p = enable_kd;
28244 BOOL use_ki_p = enable_ki;
28245 BOOL use_smooth_p = enable_smooth;
28246 BOOL use_tbh_p = enable_tbh;
28247 BOOL use_ff_p = enable_ff;
28249 int tuning_data_index = gen_number - max_generation;
28250 tuning_calculation* current_gen_calc = &gen_calc[tuning_data_index];
28251 tuning_stats* current_gen_stats = &gen_stats[tuning_data_index];
28252 bgc_size_data* data = ¤t_bgc_end_data[tuning_data_index];
28254 size_t total_generation_size = data->gen_size;
28255 size_t current_bgc_fl = data->gen_fl_size;
28257 size_t current_bgc_surv_size = get_total_surv_size (gen_number);
28258 size_t current_bgc_begin_data_size = get_total_begin_data_size (gen_number);
28260 // This is usually 0 unless a GC happened where we joined at the end of sweep
28261 size_t current_alloc = get_total_servo_alloc (gen_number);
28262 //dprintf (BGC_TUNING_LOG, ("BTL%d: current fl alloc: %Id, last recorded alloc: %Id, last_bgc_end_alloc: %Id",
28263 dprintf (BGC_TUNING_LOG, ("BTL%d: en a: %Id, la: %Id, lbgca: %Id",
28264 gen_number, current_alloc, current_gen_stats->last_alloc, current_gen_calc->last_bgc_end_alloc));
28266 double current_bgc_surv_rate = (current_bgc_begin_data_size == 0) ?
28267 0 : ((double)current_bgc_surv_size * 100.0 / (double)current_bgc_begin_data_size);
28269 current_gen_stats->last_alloc_sweep_to_end = current_alloc - current_gen_stats->last_alloc;
28271 size_t gen1_index = get_current_gc_index (max_generation - 1);
28272 size_t gen2_index = get_current_gc_index (max_generation);
28274 size_t num_gen1s_since_sweep = gen1_index - gen1_index_last_bgc_sweep;
28275 size_t num_gen1s_bgc_end = gen1_index - gen1_index_last_bgc_end;
28277 size_t gen_end_size_goal = current_gen_calc->end_gen_size_goal;
28278 double gen_sweep_flr_goal = current_gen_calc->sweep_flr_goal;
28279 size_t last_gen_alloc_to_trigger = current_gen_calc->alloc_to_trigger;
28280 size_t gen_actual_alloc_to_trigger = current_gen_calc->actual_alloc_to_trigger;
28281 size_t last_gen_alloc_to_trigger_0 = current_gen_calc->alloc_to_trigger_0;
28283 double current_end_to_sweep_flr = current_gen_calc->last_bgc_flr - current_gen_calc->current_bgc_sweep_flr;
28284 bool current_sweep_above_p = (current_gen_calc->current_bgc_sweep_flr > gen_sweep_flr_goal);
28286 #ifdef SIMPLE_DPRINTF
28287 dprintf (BGC_TUNING_LOG, ("BTL%d: sflr: c %.3f (%s), p %s, palloc: %Id, aalloc %Id(%s)",
28289 current_gen_calc->current_bgc_sweep_flr,
28290 (current_sweep_above_p ? "above" : "below"),
28291 (current_gen_calc->last_sweep_above_p ? "above" : "below"),
28292 last_gen_alloc_to_trigger,
28293 current_gen_calc->actual_alloc_to_trigger,
28294 (use_this_loop_p ? "this" : "last")));
28296 dprintf (BGC_TUNING_LOG, ("BTL%d-en[g1: %Id, g2: %Id]: end fl: %Id (%Id: S-%Id, %.3f%%->%.3f%%)",
28298 gen1_index, gen2_index, current_bgc_fl,
28299 total_generation_size, current_bgc_surv_size,
28300 current_gen_stats->last_bgc_surv_rate, current_bgc_surv_rate));
28302 dprintf (BGC_TUNING_LOG, ("BTLS%d sflr: %.3f, end-start: %Id(%Id), start-sweep: %Id(%Id), sweep-end: %Id(%Id)",
28304 current_gen_calc->current_bgc_sweep_flr,
28305 (gen1_index_last_bgc_start - gen1_index_last_bgc_end), current_gen_stats->last_alloc_end_to_start,
28306 (gen1_index_last_bgc_sweep - gen1_index_last_bgc_start), current_gen_stats->last_alloc_start_to_sweep,
28307 num_gen1s_since_sweep, current_gen_stats->last_alloc_sweep_to_end));
28308 #endif //SIMPLE_DPRINTF
28310 size_t saved_alloc_to_trigger = 0;
28312 // during our calculation alloc can be negative so use double here.
28313 double current_alloc_to_trigger = 0.0;
28315 if (!fl_tuning_triggered && use_tbh_p)
28317 current_gen_calc->alloc_to_trigger_0 = current_gen_calc->actual_alloc_to_trigger;
28318 dprintf (BGC_TUNING_LOG, ("BTL%d[g1: %Id]: not in FL tuning yet, setting alloc_to_trigger_0 to %Id",
28320 gen1_index, current_gen_calc->alloc_to_trigger_0));
28323 if (fl_tuning_triggered)
28325 BOOL tuning_kd_finished_p = FALSE;
28327 // We shouldn't have an alloc_to_trigger that's > what's consumed before sweep happens.
28328 double max_alloc_to_trigger = ((double)current_bgc_fl * (100 - gen_sweep_flr_goal) / 100.0);
28329 double min_alloc_to_trigger = (double)current_bgc_fl * 0.05;
28332 if (current_gen_calc->current_bgc_sweep_flr < 0.0)
28334 dprintf (BGC_TUNING_LOG, ("BTL%d: sflr is %.3f!!! < 0, make it 0", gen_number, current_gen_calc->current_bgc_sweep_flr));
28335 current_gen_calc->current_bgc_sweep_flr = 0.0;
28338 double adjusted_above_goal_kp = above_goal_kp;
28339 double above_goal_distance = current_gen_calc->current_bgc_sweep_flr - gen_sweep_flr_goal;
28342 if (current_gen_calc->above_goal_accu_error > max_alloc_to_trigger)
28344 dprintf (BGC_TUNING_LOG, ("g%d: ae TB! %.1f->%.1f", gen_number, current_gen_calc->above_goal_accu_error, max_alloc_to_trigger));
28346 else if (current_gen_calc->above_goal_accu_error < min_alloc_to_trigger)
28348 dprintf (BGC_TUNING_LOG, ("g%d: ae TS! %.1f->%.1f", gen_number, current_gen_calc->above_goal_accu_error, min_alloc_to_trigger));
28351 current_gen_calc->above_goal_accu_error = min (max_alloc_to_trigger, current_gen_calc->above_goal_accu_error);
28352 current_gen_calc->above_goal_accu_error = max (min_alloc_to_trigger, current_gen_calc->above_goal_accu_error);
28354 double above_goal_ki_gain = above_goal_ki * above_goal_distance * current_bgc_fl;
28355 double temp_accu_error = current_gen_calc->above_goal_accu_error + above_goal_ki_gain;
28357 if ((temp_accu_error > min_alloc_to_trigger) &&
28358 (temp_accu_error < max_alloc_to_trigger))
28360 current_gen_calc->above_goal_accu_error = temp_accu_error;
28364 //dprintf (BGC_TUNING_LOG, ("alloc accu err + %.1f=%.1f, exc",
28365 dprintf (BGC_TUNING_LOG, ("g%d: aae + %.1f=%.1f, exc", gen_number,
28366 above_goal_ki_gain,
28371 // First we do the PI loop.
28373 saved_alloc_to_trigger = current_gen_calc->alloc_to_trigger;
28374 current_alloc_to_trigger = adjusted_above_goal_kp * above_goal_distance * current_bgc_fl;
28375 // la is last alloc_to_trigger, +%Id is the diff between la and the new alloc.
28376 // laa is the last actual alloc (gen_actual_alloc_to_trigger), +%Id is the diff between la and laa.
28377 dprintf (BGC_TUNING_LOG, ("BTL%d: sflr %.3f above * %.4f * %Id = %Id bytes in alloc, la: %Id(+%Id), laa: %Id(+%Id)",
28379 (current_gen_calc->current_bgc_sweep_flr - (double)gen_sweep_flr_goal),
28380 adjusted_above_goal_kp,
28382 (size_t)current_alloc_to_trigger,
28383 saved_alloc_to_trigger,
28384 (size_t)(current_alloc_to_trigger - (double)saved_alloc_to_trigger),
28385 gen_actual_alloc_to_trigger,
28386 (gen_actual_alloc_to_trigger - saved_alloc_to_trigger)));
28390 current_alloc_to_trigger += current_gen_calc->above_goal_accu_error;
28391 dprintf (BGC_TUNING_LOG, ("BTL%d: +accu err %Id=%Id",
28393 (size_t)(current_gen_calc->above_goal_accu_error),
28394 (size_t)current_alloc_to_trigger));
28400 if (current_gen_calc->last_sweep_above_p != current_sweep_above_p)
28402 size_t new_alloc_to_trigger_0 = (last_gen_alloc_to_trigger + last_gen_alloc_to_trigger_0) / 2;
28403 dprintf (BGC_TUNING_LOG, ("BTL%d: tbh crossed SP, setting both to %Id", gen_number, new_alloc_to_trigger_0));
28404 current_gen_calc->alloc_to_trigger_0 = new_alloc_to_trigger_0;
28405 current_gen_calc->alloc_to_trigger = new_alloc_to_trigger_0;
28408 tuning_kd_finished_p = TRUE;
28412 if (!tuning_kd_finished_p)
28416 saved_alloc_to_trigger = last_gen_alloc_to_trigger;
28417 size_t alloc_delta = saved_alloc_to_trigger - gen_actual_alloc_to_trigger;
28418 double adjust_ratio = (double)alloc_delta / (double)gen_actual_alloc_to_trigger;
28419 double saved_adjust_ratio = adjust_ratio;
28420 if (enable_gradual_d)
28422 adjust_ratio = calculate_gradual_d (adjust_ratio, above_goal_kd);
28423 dprintf (BGC_TUNING_LOG, ("BTL%d: gradual kd - reduced from %.3f to %.3f",
28424 gen_number, saved_adjust_ratio, adjust_ratio));
28428 double kd = above_goal_kd;
28429 double neg_kd = 0 - kd;
28430 if (adjust_ratio > kd) adjust_ratio = kd;
28431 if (adjust_ratio < neg_kd) adjust_ratio = neg_kd;
28432 dprintf (BGC_TUNING_LOG, ("BTL%d: kd - reduced from %.3f to %.3f",
28433 gen_number, saved_adjust_ratio, adjust_ratio));
28436 current_gen_calc->alloc_to_trigger = (size_t)((double)gen_actual_alloc_to_trigger * (1 + adjust_ratio));
28438 dprintf (BGC_TUNING_LOG, ("BTL%d: kd %.3f, reduced it to %.3f * %Id, adjust %Id->%Id",
28439 gen_number, saved_adjust_ratio,
28440 adjust_ratio, gen_actual_alloc_to_trigger,
28441 saved_alloc_to_trigger, current_gen_calc->alloc_to_trigger));
28444 if (use_smooth_p && use_this_loop_p)
28446 saved_alloc_to_trigger = current_gen_calc->alloc_to_trigger;
28447 size_t gen_smoothed_alloc_to_trigger = current_gen_calc->smoothed_alloc_to_trigger;
28448 double current_num_gen1s_smooth_factor = (num_gen1s_smooth_factor > (double)num_bgcs_since_tuning_trigger) ?
28449 (double)num_bgcs_since_tuning_trigger : num_gen1s_smooth_factor;
28450 current_gen_calc->smoothed_alloc_to_trigger = (size_t)((double)saved_alloc_to_trigger / current_num_gen1s_smooth_factor +
28451 ((double)gen_smoothed_alloc_to_trigger / current_num_gen1s_smooth_factor) * (current_num_gen1s_smooth_factor - 1.0));
28453 dprintf (BGC_TUNING_LOG, ("BTL%d: smoothed %Id / %.3f + %Id / %.3f * %.3f adjust %Id->%Id",
28454 gen_number, saved_alloc_to_trigger, current_num_gen1s_smooth_factor,
28455 gen_smoothed_alloc_to_trigger, current_num_gen1s_smooth_factor,
28456 (current_num_gen1s_smooth_factor - 1.0),
28457 saved_alloc_to_trigger, current_gen_calc->smoothed_alloc_to_trigger));
28458 current_gen_calc->alloc_to_trigger = current_gen_calc->smoothed_alloc_to_trigger;
28464 double next_end_to_sweep_flr = data->gen_flr - gen_sweep_flr_goal;
28466 if (next_end_to_sweep_flr > 0.0)
28468 saved_alloc_to_trigger = current_gen_calc->alloc_to_trigger;
28469 double ff_ratio = next_end_to_sweep_flr / current_end_to_sweep_flr - 1;
28471 if (use_this_loop_p)
28473 // if we adjust down we want ff to be bigger, so the alloc will be even smaller;
28474 // if we adjust up want ff to be smaller, so the alloc will also be smaller;
28475 // the idea is we want to be slower at increase than decrease
28476 double ff_step = above_goal_ff * 0.5;
28477 double adjusted_above_goal_ff = above_goal_ff;
28479 adjusted_above_goal_ff -= ff_step;
28481 adjusted_above_goal_ff += ff_step;
28483 double adjusted_ff_ratio = ff_ratio * adjusted_above_goal_ff;
28484 current_gen_calc->alloc_to_trigger = saved_alloc_to_trigger + (size_t)((double)saved_alloc_to_trigger * adjusted_ff_ratio);
28485 dprintf (BGC_TUNING_LOG, ("BTL%d: ff (%.3f / %.3f - 1) * %.3f = %.3f adjust %Id->%Id",
28486 gen_number, next_end_to_sweep_flr, current_end_to_sweep_flr, adjusted_above_goal_ff, adjusted_ff_ratio,
28487 saved_alloc_to_trigger, current_gen_calc->alloc_to_trigger));
28492 if (use_this_loop_p)
28494 // apply low/high caps.
28495 if (current_alloc_to_trigger > max_alloc_to_trigger)
28497 dprintf (BGC_TUNING_LOG, ("BTL%d: TB! %.1f -> %.1f",
28498 gen_number, current_alloc_to_trigger, max_alloc_to_trigger));
28499 current_alloc_to_trigger = max_alloc_to_trigger;
28502 if (current_alloc_to_trigger < min_alloc_to_trigger)
28504 dprintf (BGC_TUNING_LOG, ("BTL%d: TS! %Id -> %Id",
28505 gen_number, (ptrdiff_t)current_alloc_to_trigger, (size_t)min_alloc_to_trigger));
28506 current_alloc_to_trigger = min_alloc_to_trigger;
28509 current_gen_calc->alloc_to_trigger = (size_t)current_alloc_to_trigger;
28513 // we can't do the above comparison - we could be in the situation where
28514 // we haven't done any alloc.
28515 dprintf (BGC_TUNING_LOG, ("BTL%d: ag, revert %Id->%Id",
28516 gen_number, current_gen_calc->alloc_to_trigger, last_gen_alloc_to_trigger));
28517 current_gen_calc->alloc_to_trigger = last_gen_alloc_to_trigger;
28521 // This is only executed once to get the tuning started.
28524 size_t first_alloc = (size_t)((double)current_gen_calc->first_alloc_to_trigger * 0.75);
28525 // The initial conditions can be quite erratic so check to see if the first alloc we set was reasonable - take 5% of the FL
28526 size_t min_first_alloc = current_bgc_fl / 20;
28528 current_gen_calc->alloc_to_trigger = max (first_alloc, min_first_alloc);
28530 dprintf (BGC_TUNING_LOG, ("BTL%d[g1: %Id]: BGC end, trigger FL, set gen%d alloc to max (0.75 of first: %Id, 5%% fl: %Id), actual alloc: %Id",
28531 gen_number, gen1_index, gen_number,
28532 first_alloc, min_first_alloc,
28533 current_gen_calc->actual_alloc_to_trigger));
28536 dprintf (BGC_TUNING_LOG, ("BTL%d* %Id, %.3f, %.3f, %.3f, %.3f, %.3f, %Id, %Id, %Id, %Id",
28538 total_generation_size,
28539 current_gen_calc->current_bgc_start_flr,
28540 current_gen_calc->current_bgc_sweep_flr,
28541 current_bgc_end_data[tuning_data_index].gen_flr,
28542 current_gen_stats->last_gen_increase_flr,
28543 current_bgc_surv_rate,
28544 actual_num_gen1s_to_trigger,
28546 gen_actual_alloc_to_trigger,
28547 current_gen_calc->alloc_to_trigger));
28549 gen1_index_last_bgc_end = gen1_index;
28551 current_gen_calc->last_bgc_size = total_generation_size;
28552 current_gen_calc->last_bgc_flr = current_bgc_end_data[tuning_data_index].gen_flr;
28553 current_gen_calc->last_sweep_above_p = current_sweep_above_p;
28554 current_gen_calc->last_bgc_end_alloc = current_alloc;
28556 current_gen_stats->last_bgc_physical_size = data->gen_physical_size;
28557 current_gen_stats->last_alloc_end_to_start = 0;
28558 current_gen_stats->last_alloc_start_to_sweep = 0;
28559 current_gen_stats->last_alloc_sweep_to_end = 0;
28560 current_gen_stats->last_alloc = current_alloc;
28561 current_gen_stats->last_bgc_fl_size = current_bgc_end_data[tuning_data_index].gen_fl_size;
28562 current_gen_stats->last_bgc_surv_rate = current_bgc_surv_rate;
28563 current_gen_stats->last_gen_increase_flr = 0;
28566 // Note that in this method for the !use_this_loop_p generation we will adjust
28567 // its sweep_flr accordingly. And the inner loop will not need to know about this.
28568 void gc_heap::bgc_tuning::init_bgc_end_data (int gen_number, bool use_this_loop_p)
28570 int index = gen_number - max_generation;
28571 bgc_size_data* data = ¤t_bgc_end_data[index];
28573 size_t physical_size = get_total_generation_size (gen_number);
28574 ptrdiff_t physical_fl_size = get_total_generation_fl_size (gen_number);
28575 data->gen_actual_phys_fl_size = physical_fl_size;
28577 if (fl_tuning_triggered && !use_this_loop_p)
28579 tuning_calculation* current_gen_calc = &gen_calc[gen_number - max_generation];
28581 if (current_gen_calc->actual_alloc_to_trigger > current_gen_calc->alloc_to_trigger)
28583 dprintf (BGC_TUNING_LOG, ("BTL%d: gen alloc also exceeded %Id (la: %Id), no action",
28584 gen_number, current_gen_calc->actual_alloc_to_trigger, current_gen_calc->alloc_to_trigger));
28588 // We will deduct the missing portion from alloc to fl, simulating that we consumed it.
28589 size_t remaining_alloc = current_gen_calc->alloc_to_trigger -
28590 current_gen_calc->actual_alloc_to_trigger;
28592 // now re-calc current_bgc_sweep_flr
28593 // TODO: note that I am assuming the physical size at sweep was <= end_gen_size_goal which
28594 // not have been the case.
28595 size_t gen_size = current_gen_calc->end_gen_size_goal;
28596 double sweep_flr = current_gen_calc->current_bgc_sweep_flr;
28597 size_t sweep_fl_size = (size_t)((double)gen_size * sweep_flr / 100.0);
28599 if (sweep_fl_size < remaining_alloc)
28601 dprintf (BGC_TUNING_LOG, ("BTL%d: sweep fl %Id < remain alloc %Id", gen_number, sweep_fl_size, remaining_alloc));
28602 // TODO: this is saying that we didn't have enough fl to accommodate the
28603 // remaining alloc which is suspicious. To set remaining_alloc to
28604 // something slightly smaller is only so that we could continue with
28605 // our calculation but this is something we should look into.
28606 remaining_alloc = sweep_fl_size - (10 * 1024);
28609 size_t new_sweep_fl_size = sweep_fl_size - remaining_alloc;
28610 ptrdiff_t signed_new_sweep_fl_size = sweep_fl_size - remaining_alloc;
28612 double new_current_bgc_sweep_flr = (double)new_sweep_fl_size * 100.0 / (double)gen_size;
28613 double signed_new_current_bgc_sweep_flr = (double)signed_new_sweep_fl_size * 100.0 / (double)gen_size;
28615 dprintf (BGC_TUNING_LOG, ("BTL%d: sg: %Id(%Id), sfl: %Id->%Id(%Id)(%.3f->%.3f(%.3f)), la: %Id, aa: %Id",
28616 gen_number, gen_size, physical_size, sweep_fl_size,
28617 new_sweep_fl_size, signed_new_sweep_fl_size,
28618 sweep_flr, new_current_bgc_sweep_flr, signed_new_current_bgc_sweep_flr,
28619 current_gen_calc->alloc_to_trigger, current_gen_calc->actual_alloc_to_trigger));
28621 current_gen_calc->actual_alloc_to_trigger = current_gen_calc->alloc_to_trigger;
28622 current_gen_calc->current_bgc_sweep_flr = new_current_bgc_sweep_flr;
28624 // TODO: NOTE this is duplicated in calculate_tuning except I am not * 100.0 here.
28625 size_t current_bgc_surv_size = get_total_surv_size (gen_number);
28626 size_t current_bgc_begin_data_size = get_total_begin_data_size (gen_number);
28627 double current_bgc_surv_rate = (current_bgc_begin_data_size == 0) ?
28628 0 : ((double)current_bgc_surv_size / (double)current_bgc_begin_data_size);
28630 size_t remaining_alloc_surv = (size_t)((double)remaining_alloc * current_bgc_surv_rate);
28631 physical_fl_size -= remaining_alloc_surv;
28632 dprintf (BGC_TUNING_LOG, ("BTL%d: asfl %Id-%Id=%Id, flr %.3f->%.3f, %.3f%% s, fl %Id-%Id->%Id",
28633 gen_number, sweep_fl_size, remaining_alloc, new_sweep_fl_size,
28634 sweep_flr, current_gen_calc->current_bgc_sweep_flr,
28635 (current_bgc_surv_rate * 100.0),
28636 (physical_fl_size + remaining_alloc_surv),
28637 remaining_alloc_surv, physical_fl_size));
28641 double physical_gen_flr = (double)physical_fl_size * 100.0 / (double)physical_size;
28642 data->gen_physical_size = physical_size;
28643 data->gen_physical_fl_size = physical_fl_size;
28644 data->gen_physical_flr = physical_gen_flr;
28647 void gc_heap::bgc_tuning::calc_end_bgc_fl (int gen_number)
28649 int index = gen_number - max_generation;
28650 bgc_size_data* data = ¤t_bgc_end_data[index];
28652 tuning_calculation* current_gen_calc = &gen_calc[gen_number - max_generation];
28654 size_t virtual_size = current_gen_calc->end_gen_size_goal;
28655 size_t physical_size = data->gen_physical_size;
28656 ptrdiff_t physical_fl_size = data->gen_physical_fl_size;
28657 ptrdiff_t virtual_fl_size = (ptrdiff_t)virtual_size - (ptrdiff_t)physical_size;
28658 ptrdiff_t end_gen_fl_size = physical_fl_size + virtual_fl_size;
28660 if (end_gen_fl_size < 0)
28662 end_gen_fl_size = 0;
28665 data->gen_size = virtual_size;
28666 data->gen_fl_size = end_gen_fl_size;
28667 data->gen_flr = (double)(data->gen_fl_size) * 100.0 / (double)(data->gen_size);
28669 dprintf (BGC_TUNING_LOG, ("BTL%d: vfl: %Id, size %Id->%Id, fl %Id->%Id, flr %.3f->%.3f",
28670 gen_number, virtual_fl_size,
28671 data->gen_physical_size, data->gen_size,
28672 data->gen_physical_fl_size, data->gen_fl_size,
28673 data->gen_physical_flr, data->gen_flr));
28676 // reduce_p is for NGC2s. we want to reduce the ki so we don't overshoot.
28677 double gc_heap::bgc_tuning::calculate_ml_tuning (uint64_t current_available_physical, bool reduce_p, ptrdiff_t* _vfl_from_kp, ptrdiff_t* _vfl_from_ki)
28679 ptrdiff_t error = (ptrdiff_t)(current_available_physical - available_memory_goal);
28681 // This is questionable as gen0/1 and other processes are consuming memory
28683 size_t gen2_physical_size = current_bgc_end_data[0].gen_physical_size;
28684 size_t gen3_physical_size = current_bgc_end_data[1].gen_physical_size;
28686 double max_output = (double)(total_physical_mem - available_memory_goal -
28687 gen2_physical_size - gen3_physical_size);
28689 double error_ratio = (double)error / (double)total_physical_mem;
28691 // do we want this to contribute to the integral term?
28692 bool include_in_i_p = ((error_ratio > 0.005) || (error_ratio < -0.005));
28694 dprintf (BGC_TUNING_LOG, ("total phy %Id, mem goal: %Id, curr phy: %Id, g2 phy: %Id, g3 phy: %Id",
28695 (size_t)total_physical_mem, (size_t)available_memory_goal,
28696 (size_t)current_available_physical,
28697 gen2_physical_size, gen3_physical_size));
28698 dprintf (BGC_TUNING_LOG, ("BTL: Max output: %Id, ER %Id / %Id = %.3f, %s",
28699 (size_t)max_output,
28700 error, available_memory_goal, error_ratio,
28701 (include_in_i_p ? "inc" : "exc")));
28703 if (include_in_i_p)
28705 double error_ki = ml_ki * (double)error;
28706 double temp_accu_error = accu_error + error_ki;
28708 if ((temp_accu_error > 0) && (temp_accu_error < max_output))
28709 accu_error = temp_accu_error;
28712 //dprintf (BGC_TUNING_LOG, ("ml accu err + %Id=%Id, exc",
28713 dprintf (BGC_TUNING_LOG, ("mae + %Id=%Id, exc",
28714 (size_t)error_ki, (size_t)temp_accu_error));
28720 double saved_accu_error = accu_error;
28721 accu_error = accu_error * 2.0 / 3.0;
28722 panic_activated_p = false;
28723 accu_error_panic = 0;
28724 dprintf (BGC_TUNING_LOG, ("BTL reduced accu ki %Id->%Id", (ptrdiff_t)saved_accu_error, (ptrdiff_t)accu_error));
28727 if (panic_activated_p)
28728 accu_error_panic += (double)error;
28730 accu_error_panic = 0.0;
28732 double vfl_from_kp = (double)error * ml_kp;
28733 double total_virtual_fl_size = vfl_from_kp + accu_error;
28735 if (total_virtual_fl_size < 0)
28737 dprintf (BGC_TUNING_LOG, ("BTL vfl %Id < 0", (size_t)total_virtual_fl_size));
28738 total_virtual_fl_size = 0;
28740 else if (total_virtual_fl_size > max_output)
28742 dprintf (BGC_TUNING_LOG, ("BTL vfl %Id > max", (size_t)total_virtual_fl_size));
28743 total_virtual_fl_size = max_output;
28746 *_vfl_from_kp = (ptrdiff_t)vfl_from_kp;
28747 *_vfl_from_ki = (ptrdiff_t)accu_error;
28748 return total_virtual_fl_size;
28751 void gc_heap::bgc_tuning::set_total_gen_sizes (bool use_gen2_loop_p, bool use_gen3_loop_p)
28753 size_t gen2_physical_size = current_bgc_end_data[0].gen_physical_size;
28754 size_t gen3_physical_size = 0;
28755 ptrdiff_t gen3_virtual_fl_size = 0;
28756 gen3_physical_size = current_bgc_end_data[1].gen_physical_size;
28757 double gen2_size_ratio = (double)gen2_physical_size / ((double)gen2_physical_size + (double)gen3_physical_size);
28759 // We know how far we are from the memory load goal, assuming that the memory is only
28760 // used by gen2/3 (which is obviously not the case, but that's why we are not setting the
28761 // memory goal at 90+%. Assign the memory proportionally to them.
28763 // We use entry memory load info because that seems to be more closedly correlated to what the VMM decides
28765 uint32_t current_memory_load = settings.entry_memory_load;
28766 uint64_t current_available_physical = settings.entry_available_physical_mem;
28768 panic_activated_p = (current_memory_load >= (memory_load_goal + memory_load_goal_slack));
28770 if (panic_activated_p)
28772 dprintf (BGC_TUNING_LOG, ("BTL: exceeded slack %Id >= (%Id + %Id)",
28773 (size_t)current_memory_load, (size_t)memory_load_goal,
28774 (size_t)memory_load_goal_slack));
28777 ptrdiff_t vfl_from_kp = 0;
28778 ptrdiff_t vfl_from_ki = 0;
28779 double total_virtual_fl_size = calculate_ml_tuning (current_available_physical, false, &vfl_from_kp, &vfl_from_ki);
28781 if (use_gen2_loop_p || use_gen3_loop_p)
28783 if (use_gen2_loop_p)
28785 gen2_ratio_correction += ratio_correction_step;
28789 gen2_ratio_correction -= ratio_correction_step;
28792 dprintf (BGC_TUNING_LOG, ("BTL: rc: g2 ratio %.3f%% + %d%% = %.3f%%",
28793 (gen2_size_ratio * 100.0), (int)(gen2_ratio_correction * 100.0), ((gen2_size_ratio + gen2_ratio_correction) * 100.0)));
28795 gen2_ratio_correction = min (0.99, gen2_ratio_correction);
28796 gen2_ratio_correction = max (-0.99, gen2_ratio_correction);
28798 dprintf (BGC_TUNING_LOG, ("BTL: rc again: g2 ratio %.3f%% + %d%% = %.3f%%",
28799 (gen2_size_ratio * 100.0), (int)(gen2_ratio_correction * 100.0), ((gen2_size_ratio + gen2_ratio_correction) * 100.0)));
28801 gen2_size_ratio += gen2_ratio_correction;
28803 if (gen2_size_ratio <= 0.0)
28805 gen2_size_ratio = 0.01;
28806 dprintf (BGC_TUNING_LOG, ("BTL: rc: g2 ratio->0.01"));
28809 if (gen2_size_ratio >= 1.0)
28811 gen2_size_ratio = 0.99;
28812 dprintf (BGC_TUNING_LOG, ("BTL: rc: g2 ratio->0.99"));
28816 ptrdiff_t gen2_virtual_fl_size = (ptrdiff_t)(total_virtual_fl_size * gen2_size_ratio);
28817 gen3_virtual_fl_size = (ptrdiff_t)(total_virtual_fl_size * (1.0 - gen2_size_ratio));
28818 if (gen2_virtual_fl_size < 0)
28820 ptrdiff_t saved_gen2_virtual_fl_size = gen2_virtual_fl_size;
28821 ptrdiff_t half_gen2_physical_size = (ptrdiff_t)((double)gen2_physical_size * 0.5);
28822 if (-gen2_virtual_fl_size > half_gen2_physical_size)
28824 gen2_virtual_fl_size = -half_gen2_physical_size;
28827 dprintf (BGC_TUNING_LOG, ("BTL2: n_vfl %Id(%Id)->%Id", saved_gen2_virtual_fl_size, half_gen2_physical_size, gen2_virtual_fl_size));
28828 gen2_virtual_fl_size = 0;
28831 if (gen3_virtual_fl_size < 0)
28833 ptrdiff_t saved_gen3_virtual_fl_size = gen3_virtual_fl_size;
28834 ptrdiff_t half_gen3_physical_size = (ptrdiff_t)((double)gen3_physical_size * 0.5);
28835 if (-gen3_virtual_fl_size > half_gen3_physical_size)
28837 gen3_virtual_fl_size = -half_gen3_physical_size;
28840 dprintf (BGC_TUNING_LOG, ("BTL3: n_vfl %Id(%Id)->%Id", saved_gen3_virtual_fl_size, half_gen3_physical_size, gen3_virtual_fl_size));
28841 gen3_virtual_fl_size = 0;
28844 gen_calc[0].end_gen_size_goal = gen2_physical_size + gen2_virtual_fl_size;
28845 gen_calc[1].end_gen_size_goal = gen3_physical_size + gen3_virtual_fl_size;
28847 // We calculate the end info here because the ff in fl servo loop is using this.
28848 calc_end_bgc_fl (max_generation);
28849 calc_end_bgc_fl (loh_generation);
28851 #ifdef SIMPLE_DPRINTF
28852 dprintf (BGC_TUNING_LOG, ("BTL: ml: %d (g: %d)(%s), a: %I64d (g: %I64d, elg: %Id+%Id=%Id, %Id+%Id=%Id, pi=%Id), vfl: %Id=%Id+%Id",
28853 current_memory_load, memory_load_goal,
28854 ((current_available_physical > available_memory_goal) ? "above" : "below"),
28855 current_available_physical, available_memory_goal,
28856 gen2_physical_size, gen2_virtual_fl_size, gen_calc[0].end_gen_size_goal,
28857 gen3_physical_size, gen3_virtual_fl_size, gen_calc[1].end_gen_size_goal,
28858 (ptrdiff_t)accu_error_panic,
28859 (ptrdiff_t)total_virtual_fl_size, vfl_from_kp, vfl_from_ki));
28860 #endif //SIMPLE_DPRINTF
28863 bool gc_heap::bgc_tuning::should_trigger_ngc2()
28865 return panic_activated_p;
28868 // This is our outer ml servo loop where we calculate the control for the inner fl servo loop.
28869 void gc_heap::bgc_tuning::convert_to_fl (bool use_gen2_loop_p, bool use_gen3_loop_p)
28871 size_t current_bgc_count = full_gc_counts[gc_type_background];
28873 #ifdef MULTIPLE_HEAPS
28874 for (int i = 0; i < gc_heap::n_heaps; i++)
28876 gc_heap* hp = gc_heap::g_heaps[i];
28877 hp->bgc_maxgen_end_fl_size = generation_free_list_space (hp->generation_of (max_generation));
28880 bgc_maxgen_end_fl_size = generation_free_list_space (generation_of (max_generation));
28881 #endif //MULTIPLE_HEAPS
28883 init_bgc_end_data (max_generation, use_gen2_loop_p);
28884 init_bgc_end_data (loh_generation, use_gen3_loop_p);
28885 set_total_gen_sizes (use_gen2_loop_p, use_gen3_loop_p);
28887 dprintf (BGC_TUNING_LOG, ("BTL: gen2 %Id, fl %Id(%.3f)->%Id; gen3 %Id, fl %Id(%.3f)->%Id, %Id BGCs",
28888 current_bgc_end_data[0].gen_size, current_bgc_end_data[0].gen_fl_size,
28889 current_bgc_end_data[0].gen_flr, gen_calc[0].end_gen_size_goal,
28890 current_bgc_end_data[1].gen_size, current_bgc_end_data[1].gen_fl_size,
28891 current_bgc_end_data[1].gen_flr, gen_calc[1].end_gen_size_goal,
28892 current_bgc_count));
28895 void gc_heap::bgc_tuning::record_and_adjust_bgc_end()
28897 if (!bgc_tuning::enable_fl_tuning)
28900 uint64_t elapsed_time_so_far = GetHighPrecisionTimeStamp() - process_start_time;
28901 size_t current_gen1_index = get_current_gc_index (max_generation - 1);
28902 dprintf (BGC_TUNING_LOG, ("BTL: g2t[en][g1 %Id]: %0.3f minutes",
28903 current_gen1_index,
28904 (double)elapsed_time_so_far / (double)1000000 / (double)60));
28906 if (fl_tuning_triggered)
28908 num_bgcs_since_tuning_trigger++;
28911 bool use_gen2_loop_p = (settings.reason == reason_bgc_tuning_soh);
28912 bool use_gen3_loop_p = (settings.reason == reason_bgc_tuning_loh);
28913 dprintf (BGC_TUNING_LOG, ("BTL: reason: %d, gen2 loop: %s; gen3 loop: %s, promoted %Id bytes",
28914 (((settings.reason != reason_bgc_tuning_soh) && (settings.reason != reason_bgc_tuning_loh)) ?
28915 saved_bgc_tuning_reason : settings.reason),
28916 (use_gen2_loop_p ? "yes" : "no"),
28917 (use_gen3_loop_p ? "yes" : "no"),
28918 get_total_bgc_promoted()));
28920 convert_to_fl (use_gen2_loop_p, use_gen3_loop_p);
28922 calculate_tuning (max_generation, true);
28924 if (total_loh_a_last_bgc > 0)
28926 calculate_tuning (loh_generation, true);
28930 dprintf (BGC_TUNING_LOG, ("BTL: gen3 not allocated"));
28935 next_bgc_p = false;
28936 fl_tuning_triggered = true;
28937 dprintf (BGC_TUNING_LOG, ("BTL: FL tuning ENABLED!!!"));
28940 saved_bgc_tuning_reason = -1;
28942 #endif //BGC_SERVO_TUNING
28943 #endif //BACKGROUND_GC
28945 //Clear the cards [start_card, end_card[
28946 void gc_heap::clear_cards (size_t start_card, size_t end_card)
28948 if (start_card < end_card)
28950 size_t start_word = card_word (start_card);
28951 size_t end_word = card_word (end_card);
28952 if (start_word < end_word)
28954 // Figure out the bit positions of the cards within their words
28955 unsigned bits = card_bit (start_card);
28956 card_table [start_word] &= lowbits (~0, bits);
28957 for (size_t i = start_word+1; i < end_word; i++)
28958 card_table [i] = 0;
28959 bits = card_bit (end_card);
28960 // Don't write beyond end_card (and possibly uncommitted card table space).
28963 card_table [end_word] &= highbits (~0, bits);
28968 // If the start and end cards are in the same word, just clear the appropriate card
28969 // bits in that word.
28970 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
28971 highbits (~0, card_bit (end_card)));
28973 #ifdef VERYSLOWDEBUG
28974 size_t card = start_card;
28975 while (card < end_card)
28977 assert (! (card_set_p (card)));
28980 #endif //VERYSLOWDEBUG
28981 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
28982 start_card, (size_t)card_address (start_card),
28983 end_card, (size_t)card_address (end_card)));
28987 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
28989 size_t start_card = card_of (align_on_card (start_address));
28990 size_t end_card = card_of (align_lower_card (end_address));
28991 clear_cards (start_card, end_card);
28994 // copy [srccard, ...[ to [dst_card, end_card[
28995 // This will set the same bit twice. Can be optimized.
28997 void gc_heap::copy_cards (size_t dst_card,
29002 // If the range is empty, this function is a no-op - with the subtlety that
29003 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
29004 // outside the committed region. To avoid the access, leave early.
29005 if (!(dst_card < end_card))
29008 unsigned int srcbit = card_bit (src_card);
29009 unsigned int dstbit = card_bit (dst_card);
29010 size_t srcwrd = card_word (src_card);
29011 size_t dstwrd = card_word (dst_card);
29012 unsigned int srctmp = card_table[srcwrd];
29013 unsigned int dsttmp = card_table[dstwrd];
29015 for (size_t card = dst_card; card < end_card; card++)
29017 if (srctmp & (1 << srcbit))
29018 dsttmp |= 1 << dstbit;
29020 dsttmp &= ~(1 << dstbit);
29021 if (!(++srcbit % 32))
29023 srctmp = card_table[++srcwrd];
29029 if (srctmp & (1 << srcbit))
29030 dsttmp |= 1 << dstbit;
29033 if (!(++dstbit % 32))
29035 card_table[dstwrd] = dsttmp;
29037 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
29040 card_bundle_set(cardw_card_bundle(dstwrd));
29045 dsttmp = card_table[dstwrd];
29050 card_table[dstwrd] = dsttmp;
29052 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
29055 card_bundle_set(cardw_card_bundle(dstwrd));
29060 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
29062 ptrdiff_t relocation_distance = src - dest;
29063 size_t start_dest_card = card_of (align_on_card (dest));
29064 size_t end_dest_card = card_of (dest + len - 1);
29065 size_t dest_card = start_dest_card;
29066 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
29067 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
29068 src_card, (size_t)src, dest_card, (size_t)dest));
29069 dprintf (3,(" %Ix->%Ix:%Ix[",
29070 (size_t)src+len, end_dest_card, (size_t)dest+len));
29072 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
29073 dest, src, len, relocation_distance, (align_on_card (dest))));
29075 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
29076 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
29078 //First card has two boundaries
29079 if (start_dest_card != card_of (dest))
29081 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
29082 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
29084 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
29085 (card_address (start_dest_card) + relocation_distance),
29086 card_of (card_address (start_dest_card) + relocation_distance),
29088 card_of (src + len - 1)));
29090 dprintf (3, ("setting card: %Ix", card_of (dest)));
29091 set_card (card_of (dest));
29095 if (card_set_p (card_of (src)))
29096 set_card (card_of (dest));
29099 copy_cards (dest_card, src_card, end_dest_card,
29100 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
29102 //Last card has two boundaries.
29103 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
29104 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
29106 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
29107 (card_address (end_dest_card) + relocation_distance),
29108 card_of (card_address (end_dest_card) + relocation_distance),
29112 dprintf (3, ("setting card: %Ix", end_dest_card));
29113 set_card (end_dest_card);
29116 if (card_set_p (card_of (src + len - 1)))
29117 set_card (end_dest_card);
29119 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
29120 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
29124 #ifdef BACKGROUND_GC
29125 // this does not need the Interlocked version of mark_array_set_marked.
29126 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
29128 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
29129 (size_t)src, (size_t)dest,
29130 (size_t)src+len, (size_t)dest+len));
29132 uint8_t* src_o = src;
29134 uint8_t* src_end = src + len;
29135 int align_const = get_alignment_constant (TRUE);
29136 ptrdiff_t reloc = dest - src;
29138 while (src_o < src_end)
29140 uint8_t* next_o = src_o + Align (size (src_o), align_const);
29142 if (background_object_marked (src_o, TRUE))
29144 dest_o = src_o + reloc;
29146 //if (background_object_marked (dest_o, FALSE))
29148 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
29149 // FATAL_GC_ERROR();
29152 background_mark (dest_o,
29153 background_saved_lowest_address,
29154 background_saved_highest_address);
29155 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
29161 #endif //BACKGROUND_GC
29163 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
29165 size_t new_current_brick = brick_of (o);
29166 set_brick (new_current_brick,
29167 (o - brick_address (new_current_brick)));
29168 size_t b = 1 + new_current_brick;
29169 size_t limit = brick_of (next_o);
29170 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
29171 dprintf(3,("b:%Ix->%Ix-%Ix",
29172 new_current_brick, (size_t)o, (size_t)next_o));
29175 set_brick (b,(new_current_brick - b));
29180 // start can not be >= heap_segment_allocated for the segment.
29181 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
29183 size_t brick = brick_of (start);
29185 //last_object == null -> no search shortcut needed
29186 if ((brick == brick_of (first_object) || (start <= first_object)))
29192 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
29193 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
29194 int brick_entry = 0;
29197 if (prev_brick < min_brick)
29201 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
29205 assert (! ((brick_entry == 0)));
29206 prev_brick = (brick_entry + prev_brick);
29209 o = ((prev_brick < min_brick) ? first_object :
29210 brick_address (prev_brick) + brick_entry - 1);
29211 assert (o <= start);
29214 assert (Align (size (o)) >= Align (min_obj_size));
29215 uint8_t* next_o = o + Align (size (o));
29216 size_t curr_cl = (size_t)next_o / brick_size;
29217 size_t min_cl = (size_t)first_object / brick_size;
29220 unsigned int n_o = 1;
29223 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
29225 while (next_o <= start)
29233 assert (Align (size (o)) >= Align (min_obj_size));
29234 next_o = o + Align (size (o));
29236 }while (next_o < next_b);
29238 if (((size_t)next_o / brick_size) != curr_cl)
29240 if (curr_cl >= min_cl)
29242 fix_brick_to_highest (o, next_o);
29244 curr_cl = (size_t) next_o / brick_size;
29246 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
29249 size_t bo = brick_of (o);
29250 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
29251 dprintf (3, ("%Id o, [%Ix-[%Ix",
29255 set_brick (bo, (o - brick_address(bo)));
29270 // Find the first non-zero card word between cardw and cardw_end.
29271 // The index of the word we find is returned in cardw.
29272 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
29274 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
29275 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
29277 if (card_bundles_enabled())
29279 size_t cardb = cardw_card_bundle (cardw);
29280 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
29283 // Find a non-zero bundle
29284 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
29288 if (cardb == end_cardb)
29291 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
29292 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
29293 while ((card_word < card_word_end) && !(*card_word))
29298 if (card_word != card_word_end)
29300 cardw = (card_word - &card_table[0]);
29303 else if ((cardw <= card_bundle_cardw (cardb)) &&
29304 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
29306 // a whole bundle was explored and is empty
29307 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
29308 dd_collection_count (dynamic_data_of (0)),
29309 cardb, card_bundle_cardw (cardb),
29310 card_bundle_cardw (cardb+1)));
29311 card_bundle_clear (cardb);
29319 uint32_t* card_word = &card_table[cardw];
29320 uint32_t* card_word_end = &card_table [cardw_end];
29322 while (card_word < card_word_end)
29324 if ((*card_word) != 0)
29326 cardw = (card_word - &card_table [0]);
29338 #endif //CARD_BUNDLE
29340 // Find cards that are set between two points in a card table.
29342 // card_table : The card table.
29343 // card : [in/out] As input, the card to start searching from.
29344 // As output, the first card that's set.
29345 // card_word_end : The card word at which to stop looking.
29346 // end_card : [out] The last card which is set.
29347 BOOL gc_heap::find_card(uint32_t* card_table,
29349 size_t card_word_end,
29352 uint32_t* last_card_word;
29353 uint32_t card_word_value;
29354 uint32_t bit_position;
29356 if (card_word (card) >= card_word_end)
29359 // Find the first card which is set
29360 last_card_word = &card_table [card_word (card)];
29361 bit_position = card_bit (card);
29362 card_word_value = (*last_card_word) >> bit_position;
29363 if (!card_word_value)
29367 // Using the card bundle, go through the remaining card words between here and
29368 // card_word_end until we find one that is non-zero.
29369 size_t lcw = card_word(card) + 1;
29370 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
29376 last_card_word = &card_table [lcw];
29377 card_word_value = *last_card_word;
29380 #else //CARD_BUNDLE
29381 // Go through the remaining card words between here and card_word_end until we find
29382 // one that is non-zero.
29388 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
29389 if (last_card_word < &card_table [card_word_end])
29391 card_word_value = *last_card_word;
29395 // We failed to find any non-zero card words before we got to card_word_end
29398 #endif //CARD_BUNDLE
29402 // Look for the lowest bit set
29403 if (card_word_value)
29405 while (!(card_word_value & 1))
29408 card_word_value = card_word_value / 2;
29412 // card is the card word index * card size + the bit index within the card
29413 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
29417 // Keep going until we get to an un-set card.
29419 card_word_value = card_word_value / 2;
29421 // If we reach the end of the card word and haven't hit a 0 yet, start going
29422 // card word by card word until we get to one that's not fully set (0xFFFF...)
29423 // or we reach card_word_end.
29424 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end-1]))
29428 card_word_value = *(++last_card_word);
29429 } while ((last_card_word < &card_table [card_word_end-1]) &&
29430 (card_word_value == ~0u /* (1 << card_word_width)-1 */));
29433 } while (card_word_value & 1);
29435 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
29437 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
29438 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
29443 //because of heap expansion, computing end is complicated.
29444 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
29446 if ((low >= heap_segment_mem (seg)) &&
29447 (low < heap_segment_allocated (seg)))
29450 return heap_segment_allocated (seg);
29454 gc_heap::compute_next_boundary (int gen_number,
29457 //when relocating, the fault line is the plan start of the younger
29458 //generation because the generation is promoted.
29459 if (relocating && (gen_number == (settings.condemned_generation + 1)))
29461 generation* gen = generation_of (gen_number - 1);
29462 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
29463 assert (gen_alloc);
29468 assert (gen_number > settings.condemned_generation);
29469 return generation_allocation_start (generation_of (gen_number - 1 ));
29475 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
29476 size_t& cg_pointers_found)
29478 if ((gc_low <= o) && (gc_high > o))
29482 #ifdef MULTIPLE_HEAPS
29485 gc_heap* hp = heap_of (o);
29488 if ((hp->gc_low <= o) &&
29495 #endif //MULTIPLE_HEAPS
29496 cg_pointers_found ++;
29497 dprintf (4, ("keep card live for %Ix", o));
29501 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
29502 size_t& cg_pointers_found,
29503 card_fn fn, uint8_t* nhigh,
29504 uint8_t* next_boundary
29505 CARD_MARKING_STEALING_ARG(gc_heap* hpt))
29507 #if defined(FEATURE_CARD_MARKING_STEALING) && defined(MULTIPLE_HEAPS)
29508 int thread = hpt->heap_number;
29511 #if defined (MULTIPLE_HEAPS)
29512 gc_heap* hpt = this;
29515 if ((gc_low <= *poo) && (gc_high > *poo))
29518 call_fn(hpt,fn) (poo THREAD_NUMBER_ARG);
29520 #ifdef MULTIPLE_HEAPS
29523 gc_heap* hp = heap_of_gc (*poo);
29526 if ((hp->gc_low <= *poo) &&
29527 (hp->gc_high > *poo))
29530 call_fn(hpt,fn) (poo THREAD_NUMBER_ARG);
29532 if ((fn == &gc_heap::relocate_address) ||
29533 ((hp->ephemeral_low <= *poo) &&
29534 (hp->ephemeral_high > *poo)))
29536 cg_pointers_found++;
29540 #endif //MULTIPLE_HEAPS
29541 if ((next_boundary <= *poo) && (nhigh > *poo))
29543 cg_pointers_found ++;
29544 dprintf (4, ("cg pointer %Ix found, %Id so far",
29545 (size_t)*poo, cg_pointers_found ));
29550 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
29551 size_t& cg_pointers_found,
29552 size_t& n_eph, size_t& n_card_set,
29553 size_t& card, size_t& end_card,
29554 BOOL& foundp, uint8_t*& start_address,
29555 uint8_t*& limit, size_t& n_cards_cleared
29556 CARD_MARKING_STEALING_ARGS(card_marking_enumerator& card_mark_enumerator, heap_segment* seg, size_t &card_word_end_out))
29558 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
29559 dprintf (3, ("ct: %Id cg", cg_pointers_found));
29560 BOOL passed_end_card_p = FALSE;
29563 if (cg_pointers_found == 0)
29565 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
29566 dprintf(3,(" CC [%Ix, %Ix[ ",
29567 (size_t)card_address(card), (size_t)po));
29568 clear_cards (card, card_of(po));
29569 n_card_set -= (card_of (po) - card);
29570 n_cards_cleared += (card_of (po) - card);
29573 n_eph +=cg_pointers_found;
29574 cg_pointers_found = 0;
29575 card = card_of (po);
29576 if (card >= end_card)
29578 passed_end_card_p = TRUE;
29579 dprintf (3, ("card %Ix exceeding end_card %Ix",
29580 (size_t)card, (size_t)end_card));
29581 foundp = find_card (card_table, card, card_word_end, end_card);
29584 n_card_set+= end_card - card;
29585 start_address = card_address (card);
29586 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
29587 (size_t)card, (size_t)start_address,
29588 (size_t)card_address (end_card)));
29590 limit = min (end, card_address (end_card));
29592 #ifdef FEATURE_CARD_MARKING_STEALING
29593 // the card bit @ end_card should not be set
29594 // if end_card is still shy of the limit set by card_word_end
29595 assert(!((card_word(end_card) < card_word_end) &&
29596 card_set_p(end_card)));
29599 card_word_end_out = 0;
29600 foundp = find_next_chunk(card_mark_enumerator, seg, n_card_set, start_address, limit, card, end_card, card_word_end_out);
29603 // the card bit @ end_card should not be set -
29604 // find_card is supposed to terminate only when it finds a 0 bit
29605 // or the end of the segment
29606 assert (!((limit < end) &&
29607 card_set_p (end_card)));
29611 return passed_end_card_p;
29614 #ifdef FEATURE_CARD_MARKING_STEALING
29615 bool card_marking_enumerator::move_next(heap_segment* seg, uint8_t*& low, uint8_t*& high)
29617 if (segment == nullptr)
29620 uint32_t chunk_index = old_chunk_index;
29621 old_chunk_index = INVALID_CHUNK_INDEX;
29622 if (chunk_index == INVALID_CHUNK_INDEX)
29623 chunk_index = Interlocked::Increment((volatile int32_t *)chunk_index_counter);
29627 uint32_t chunk_index_within_seg = chunk_index - segment_start_chunk_index;
29629 uint8_t* start = heap_segment_mem(segment);
29630 uint8_t* end = compute_next_end(segment, gc_low);
29632 uint8_t* aligned_start = (uint8_t*)((size_t)start & ~(CARD_MARKING_STEALING_GRANULARITY - 1));
29633 size_t seg_size = end - aligned_start;
29634 uint32_t chunk_count_within_seg = (uint32_t)((seg_size + (CARD_MARKING_STEALING_GRANULARITY - 1)) / CARD_MARKING_STEALING_GRANULARITY);
29635 if (chunk_index_within_seg < chunk_count_within_seg)
29637 if (seg == segment)
29639 low = (chunk_index_within_seg == 0) ? start : (aligned_start + (size_t)chunk_index_within_seg * CARD_MARKING_STEALING_GRANULARITY);
29640 high = (chunk_index_within_seg + 1 == chunk_count_within_seg) ? end : (aligned_start + (size_t)(chunk_index_within_seg + 1) * CARD_MARKING_STEALING_GRANULARITY);
29646 // we found the correct segment, but it's not the segment our caller is in
29648 // our caller should still be in one of the previous segments
29650 for (heap_segment* cur_seg = seg; cur_seg != segment; cur_seg = heap_segment_next_in_range(cur_seg))
29656 // keep the chunk index for later
29657 old_chunk_index = chunk_index;
29662 segment = heap_segment_next_in_range(segment);
29663 if (segment == nullptr)
29667 segment_start_chunk_index += chunk_count_within_seg;
29671 bool gc_heap::find_next_chunk(card_marking_enumerator& card_mark_enumerator, heap_segment* seg, size_t& n_card_set,
29672 uint8_t*& start_address, uint8_t*& limit,
29673 size_t& card, size_t& end_card, size_t& card_word_end)
29677 if (card_word_end != 0 && find_card(card_table, card, card_word_end, end_card))
29679 assert(end_card <= card_word_end * card_word_width);
29680 n_card_set += end_card - card;
29681 start_address = card_address(card);
29682 dprintf(3, ("NewC: %Ix, start: %Ix, end: %Ix",
29683 (size_t)card, (size_t)start_address,
29684 (size_t)card_address(end_card)));
29685 limit = min(card_mark_enumerator.get_chunk_high(), card_address(end_card));
29686 dprintf (3, ("New run of cards on heap %d: [%Ix,%Ix[", heap_number, (size_t)start_address, (size_t)limit));
29689 // we have exhausted this chunk, get the next one
29690 uint8_t* chunk_low = nullptr;
29691 uint8_t* chunk_high = nullptr;
29692 if (!card_mark_enumerator.move_next(seg, chunk_low, chunk_high))
29694 dprintf (3, ("No more chunks on heap %d\n", heap_number));
29697 card = card_of (chunk_low);
29698 card_word_end = (card_of(align_on_card_word(chunk_high)) / card_word_width);
29699 dprintf (3, ("Moved to next chunk on heap %d: [%Ix,%Ix[", heap_number, (size_t)chunk_low, (size_t)chunk_high));
29702 #endif // FEATURE_CARD_MARKING_STEALING
29704 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating CARD_MARKING_STEALING_ARG(gc_heap* hpt))
29706 #ifdef BACKGROUND_GC
29707 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
29708 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
29710 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
29711 PREFIX_ASSUME(soh_seg != NULL);
29715 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
29717 heap_segment_background_allocated (soh_seg),
29718 heap_segment_allocated (soh_seg)));
29720 soh_seg = heap_segment_next_rw (soh_seg);
29722 #endif //BACKGROUND_GC
29724 uint8_t* low = gc_low;
29725 uint8_t* high = gc_high;
29726 size_t end_card = 0;
29728 generation* oldest_gen = generation_of (max_generation);
29729 int curr_gen_number = max_generation;
29730 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
29731 uint8_t* next_boundary = compute_next_boundary(curr_gen_number, relocating);
29733 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
29734 PREFIX_ASSUME(seg != NULL);
29736 uint8_t* beg = generation_allocation_start (oldest_gen);
29737 uint8_t* end = compute_next_end (seg, low);
29738 uint8_t* last_object = beg;
29740 size_t cg_pointers_found = 0;
29742 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
29746 size_t n_card_set = 0;
29747 uint8_t* nhigh = (relocating ?
29748 heap_segment_plan_allocated (ephemeral_heap_segment) : high);
29750 BOOL foundp = FALSE;
29751 uint8_t* start_address = 0;
29752 uint8_t* limit = 0;
29753 size_t card = card_of (beg);
29754 #ifdef BACKGROUND_GC
29755 BOOL consider_bgc_mark_p = FALSE;
29756 BOOL check_current_sweep_p = FALSE;
29757 BOOL check_saved_sweep_p = FALSE;
29758 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
29759 #endif //BACKGROUND_GC
29761 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
29762 size_t total_cards_cleared = 0;
29764 #ifdef FEATURE_CARD_MARKING_STEALING
29765 card_marking_enumerator card_mark_enumerator (seg, low, (VOLATILE(uint32_t)*)&card_mark_chunk_index_soh);
29767 #endif // FEATURE_CARD_MARKING_STEALING
29771 if (card_of(last_object) > card)
29773 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
29774 if (cg_pointers_found == 0)
29776 uint8_t* last_object_processed = last_object;
29777 #ifdef FEATURE_CARD_MARKING_STEALING
29778 last_object_processed = min(limit, last_object);
29779 #endif // FEATURE_CARD_MARKING_STEALING
29780 dprintf (3, (" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object_processed));
29781 clear_cards(card, card_of(last_object_processed));
29782 n_card_set -= (card_of(last_object_processed) - card);
29783 total_cards_cleared += (card_of(last_object_processed) - card);
29786 n_eph += cg_pointers_found;
29787 cg_pointers_found = 0;
29788 card = card_of (last_object);
29791 if (card >= end_card)
29793 #ifdef FEATURE_CARD_MARKING_STEALING
29794 // find another chunk with some cards set
29795 foundp = find_next_chunk(card_mark_enumerator, seg, n_card_set, start_address, limit, card, end_card, card_word_end);
29796 #else // FEATURE_CARD_MARKING_STEALING
29797 foundp = find_card(card_table, card, card_word_end, end_card);
29800 n_card_set += end_card - card;
29801 start_address = max (beg, card_address (card));
29803 limit = min (end, card_address (end_card));
29804 #endif // FEATURE_CARD_MARKING_STEALING
29806 if (!foundp || (last_object >= end) || (card_address (card) >= end))
29808 if (foundp && (cg_pointers_found == 0))
29810 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
29812 clear_cards (card, card_of (end));
29813 n_card_set -= (card_of (end) - card);
29814 total_cards_cleared += (card_of (end) - card);
29816 n_eph += cg_pointers_found;
29817 cg_pointers_found = 0;
29818 #ifdef FEATURE_CARD_MARKING_STEALING
29819 // we have decided to move to the next segment - make sure we exhaust the chunk enumerator for this segment
29820 card_mark_enumerator.exhaust_segment(seg);
29821 #endif // FEATURE_CARD_MARKING_STEALING
29822 if ((seg = heap_segment_next_in_range (seg)) != 0)
29824 #ifdef BACKGROUND_GC
29825 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
29826 #endif //BACKGROUND_GC
29827 beg = heap_segment_mem (seg);
29828 end = compute_next_end (seg, low);
29829 #ifdef FEATURE_CARD_MARKING_STEALING
29831 #else // FEATURE_CARD_MARKING_STEALING
29832 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
29833 #endif // FEATURE_CARD_MARKING_STEALING
29834 card = card_of (beg);
29845 assert (card_set_p (card));
29847 uint8_t* o = last_object;
29849 o = find_first_object (start_address, last_object);
29850 // Never visit an object twice.
29851 assert (o >= last_object);
29853 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
29854 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
29855 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
29859 assert (Align (size (o)) >= Align (min_obj_size));
29860 size_t s = size (o);
29862 // next_o is the next object in the heap walk
29863 uint8_t* next_o = o + Align (s);
29865 // while cont_o is the object we should continue with at the end_object label
29866 uint8_t* cont_o = next_o;
29870 if ((o >= gen_boundary) &&
29871 (seg == ephemeral_heap_segment))
29873 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
29875 assert ((curr_gen_number > 0));
29876 gen_boundary = generation_allocation_start
29877 (generation_of (curr_gen_number - 1));
29878 next_boundary = (compute_next_boundary
29879 (curr_gen_number, relocating));
29882 dprintf (4, ("|%Ix|", (size_t)o));
29884 if (next_o < start_address)
29889 #ifdef BACKGROUND_GC
29890 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
29894 #endif //BACKGROUND_GC
29896 #ifdef COLLECTIBLE_CLASS
29897 if (is_collectible(o))
29899 BOOL passed_end_card_p = FALSE;
29901 if (card_of (o) > card)
29903 passed_end_card_p = card_transition (o, end, card_word_end,
29907 foundp, start_address,
29908 limit, total_cards_cleared
29909 CARD_MARKING_STEALING_ARGS(card_mark_enumerator, seg, card_word_end));
29912 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
29914 // card is valid and it covers the head of the object
29915 if (fn == &gc_heap::relocate_address)
29917 keep_card_live (o, n_gen, cg_pointers_found);
29921 uint8_t* class_obj = get_class_object (o);
29922 mark_through_cards_helper (&class_obj, n_gen,
29923 cg_pointers_found, fn,
29924 nhigh, next_boundary CARD_MARKING_STEALING_ARG(hpt));
29928 if (passed_end_card_p)
29930 if (foundp && (card_address (card) < next_o))
29932 goto go_through_refs;
29934 else if (foundp && (start_address < limit))
29936 cont_o = find_first_object (start_address, o);
29945 #endif //COLLECTIBLE_CLASS
29947 if (contain_pointers (o))
29949 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
29952 dprintf (4, ("normal object path"));
29954 (method_table(o), o, s, poo,
29955 start_address, use_start, (o + s),
29957 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
29958 if (card_of ((uint8_t*)poo) > card)
29960 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
29965 foundp, start_address,
29966 limit, total_cards_cleared
29967 CARD_MARKING_STEALING_ARGS(card_mark_enumerator, seg, card_word_end));
29969 if (passed_end_card_p)
29971 if (foundp && (card_address (card) < next_o))
29975 if (ppstop <= (uint8_t**)start_address)
29977 else if (poo < (uint8_t**)start_address)
29978 {poo = (uint8_t**)start_address;}
29981 else if (foundp && (start_address < limit))
29983 cont_o = find_first_object (start_address, o);
29991 mark_through_cards_helper (poo, n_gen,
29992 cg_pointers_found, fn,
29993 nhigh, next_boundary CARD_MARKING_STEALING_ARG(hpt));
30000 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
30002 if (brick_table [brick_of (o)] <0)
30003 fix_brick_to_highest (o, next_o);
30011 // compute the efficiency ratio of the card table
30014 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
30015 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
30016 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
30020 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
30021 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
30025 #ifdef SEG_REUSE_STATS
30026 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
30028 size_t total_items = 0;
30030 for (int i = 0; i < count; i++)
30032 total_items += ordered_indices[i];
30033 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
30034 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
30036 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
30037 return total_items;
30039 #endif // SEG_REUSE_STATS
30041 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
30043 // detect pinned plugs
30044 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
30046 deque_pinned_plug();
30047 update_oldest_pinned_plug();
30048 dprintf (3, ("deque pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
30052 size_t plug_size = last_plug_size + Align(min_obj_size);
30053 BOOL is_padded = FALSE;
30056 plug_size += Align (min_obj_size);
30058 #endif //SHORT_PLUGS
30060 #ifdef RESPECT_LARGE_ALIGNMENT
30061 plug_size += switch_alignment_size (is_padded);
30062 #endif //RESPECT_LARGE_ALIGNMENT
30064 total_ephemeral_plugs += plug_size;
30065 size_t plug_size_power2 = round_up_power2 (plug_size);
30066 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
30067 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
30071 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
30075 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
30077 assert ((tree != NULL));
30078 if (node_left_child (tree))
30080 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
30083 if (last_plug != 0)
30085 uint8_t* plug = tree;
30086 size_t gap_size = node_gap_size (plug);
30087 uint8_t* gap = (plug - gap_size);
30088 uint8_t* last_plug_end = gap;
30089 size_t last_plug_size = (last_plug_end - last_plug);
30090 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
30091 tree, last_plug, gap_size, gap, last_plug_size));
30093 if (tree == oldest_pinned_plug)
30095 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
30096 tree, last_plug, last_plug_size));
30097 mark* m = oldest_pin();
30098 if (m->has_pre_plug_info())
30100 last_plug_size += sizeof (gap_reloc_pair);
30101 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
30104 // Can't assert here - if it's a pinned plug it can be less.
30105 //assert (last_plug_size >= Align (min_obj_size));
30107 count_plug (last_plug_size, last_plug);
30112 if (node_right_child (tree))
30114 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
30118 void gc_heap::build_ordered_plug_indices ()
30120 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
30121 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
30123 uint8_t* start_address = generation_limit (max_generation);
30124 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
30125 size_t current_brick = brick_of (start_address);
30126 size_t end_brick = brick_of (end_address - 1);
30127 uint8_t* last_plug = 0;
30129 //Look for the right pinned plug to start from.
30130 reset_pinned_queue_bos();
30131 while (!pinned_plug_que_empty_p())
30133 mark* m = oldest_pin();
30134 if ((m->first >= start_address) && (m->first < end_address))
30136 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
30141 deque_pinned_plug();
30144 update_oldest_pinned_plug();
30146 while (current_brick <= end_brick)
30148 int brick_entry = brick_table [ current_brick ];
30149 if (brick_entry >= 0)
30151 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
30159 count_plug (end_address - last_plug, last_plug);
30162 // we need to make sure that after fitting all the existing plugs, we
30163 // have big enough free space left to guarantee that the next allocation
30165 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
30166 total_ephemeral_plugs += extra_size;
30167 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
30168 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
30170 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
30172 #ifdef SEG_REUSE_STATS
30173 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
30174 size_t total_plug_power2 = 0;
30175 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
30176 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
30177 total_ephemeral_plugs,
30179 (total_ephemeral_plugs ?
30180 (total_plug_power2 * 100 / total_ephemeral_plugs) :
30182 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
30183 #endif // SEG_REUSE_STATS
30186 void gc_heap::init_ordered_free_space_indices ()
30188 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
30189 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
30192 void gc_heap::trim_free_spaces_indices ()
30194 trimmed_free_space_index = -1;
30195 size_t max_count = max_free_space_items - 1;
30198 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
30200 count += ordered_free_space_indices[i];
30202 if (count >= max_count)
30208 ptrdiff_t extra_free_space_items = count - max_count;
30210 if (extra_free_space_items > 0)
30212 ordered_free_space_indices[i] -= extra_free_space_items;
30213 free_space_items = max_count;
30214 trimmed_free_space_index = i;
30218 free_space_items = count;
30226 free_space_buckets = MAX_NUM_BUCKETS - i;
30228 for (--i; i >= 0; i--)
30230 ordered_free_space_indices[i] = 0;
30233 memcpy (saved_ordered_free_space_indices,
30234 ordered_free_space_indices,
30235 sizeof(ordered_free_space_indices));
30238 // We fit as many plugs as we can and update the number of plugs left and the number
30239 // of free spaces left.
30240 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
30242 assert (small_index <= big_index);
30243 assert (big_index < MAX_NUM_BUCKETS);
30245 size_t small_blocks = ordered_blocks[small_index];
30247 if (small_blocks == 0)
30252 size_t big_spaces = ordered_spaces[big_index];
30254 if (big_spaces == 0)
30259 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
30261 small_blocks, (small_index + MIN_INDEX_POWER2),
30262 big_spaces, (big_index + MIN_INDEX_POWER2)));
30264 size_t big_to_small = big_spaces << (big_index - small_index);
30266 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
30267 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
30269 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
30270 BOOL can_fit = (extra_small_spaces >= 0);
30274 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
30276 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
30281 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
30282 ordered_spaces[big_index] = 0;
30283 if (extra_small_spaces > 0)
30285 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
30286 ordered_blocks[small_index] = 0;
30287 for (i = small_index; i < big_index; i++)
30289 if (extra_small_spaces & 1)
30291 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
30293 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
30294 ordered_spaces[i] += 1;
30296 extra_small_spaces >>= 1;
30299 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
30301 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
30302 ordered_spaces[i] += extra_small_spaces;
30306 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
30308 (small_index + MIN_INDEX_POWER2),
30309 ordered_blocks[small_index],
30310 (ordered_blocks[small_index] - big_to_small)));
30311 ordered_blocks[small_index] -= big_to_small;
30314 #ifdef SEG_REUSE_STATS
30316 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
30317 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
30319 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
30320 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
30321 #endif //SEG_REUSE_STATS
30326 // space_index gets updated to the biggest available space index.
30327 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
30329 assert (*space_index >= block_index);
30331 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
30334 if (*space_index < block_index)
30343 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
30345 #ifdef FEATURE_STRUCTALIGN
30346 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
30348 #endif // FEATURE_STRUCTALIGN
30349 int space_index = count - 1;
30350 for (int block_index = (count - 1); block_index >= 0; block_index--)
30352 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
30361 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
30363 assert (bestfit_seg);
30365 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
30366 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
30367 // free_space_buckets,
30368 // free_space_items);
30370 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
30371 ordered_free_space_indices,
30375 assert (settings.condemned_generation == max_generation);
30377 uint8_t* first_address = heap_segment_mem (seg);
30378 uint8_t* end_address = heap_segment_reserved (seg);
30379 //look through the pinned plugs for relevant ones.
30380 //Look for the right pinned plug to start from.
30381 reset_pinned_queue_bos();
30384 // See comment in can_expand_into_p why we need this size.
30385 size_t eph_gen_starts = eph_gen_starts_size + Align (min_obj_size);
30386 BOOL has_fit_gen_starts = FALSE;
30388 while (!pinned_plug_que_empty_p())
30391 if ((pinned_plug (m) >= first_address) &&
30392 (pinned_plug (m) < end_address) &&
30393 (pinned_len (m) >= eph_gen_starts))
30396 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
30401 deque_pinned_plug();
30405 if (!pinned_plug_que_empty_p())
30407 bestfit_seg->add ((void*)m, TRUE, TRUE);
30408 deque_pinned_plug();
30410 has_fit_gen_starts = TRUE;
30413 while (!pinned_plug_que_empty_p() &&
30414 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
30416 bestfit_seg->add ((void*)m, TRUE, FALSE);
30417 deque_pinned_plug();
30421 if (commit_end_of_seg)
30423 if (!has_fit_gen_starts)
30425 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
30427 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
30431 bestfit_seg->check();
30435 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
30437 if (!end_of_segment_p)
30439 trim_free_spaces_indices ();
30442 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
30443 ordered_free_space_indices,
30446 return can_bestfit;
30449 BOOL gc_heap::best_fit (size_t free_space,
30450 size_t largest_free_space,
30451 size_t additional_space,
30452 BOOL* use_additional_space)
30454 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
30456 assert (!additional_space || (additional_space && use_additional_space));
30457 if (use_additional_space)
30459 *use_additional_space = FALSE;
30462 if (ordered_plug_indices_init == FALSE)
30464 total_ephemeral_plugs = 0;
30465 build_ordered_plug_indices();
30466 ordered_plug_indices_init = TRUE;
30470 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
30473 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
30475 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
30476 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
30477 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
30478 if (!can_fit_empty_eph)
30480 can_fit_empty_eph = (additional_space >= empty_eph);
30482 if (can_fit_empty_eph)
30484 *use_additional_space = TRUE;
30488 return can_fit_empty_eph;
30491 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
30493 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
30497 if ((free_space + additional_space) == 0)
30499 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
30503 #ifdef SEG_REUSE_STATS
30504 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
30505 size_t total_free_space_power2 = 0;
30506 size_t total_free_space_items =
30507 dump_buckets (ordered_free_space_indices,
30509 &total_free_space_power2);
30510 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
30512 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
30513 total_ephemeral_plugs,
30515 total_free_space_power2,
30516 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
30517 additional_space));
30519 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
30520 memcpy (saved_all_free_space_indices,
30521 ordered_free_space_indices,
30522 sizeof(saved_all_free_space_indices));
30524 #endif // SEG_REUSE_STATS
30526 if (total_ephemeral_plugs > (free_space + additional_space))
30531 use_bestfit = try_best_fit(FALSE);
30533 if (!use_bestfit && additional_space)
30535 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
30537 if (relative_free_space_index != -1)
30539 int relative_plug_index = 0;
30540 size_t plugs_to_fit = 0;
30542 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
30544 plugs_to_fit = ordered_plug_indices[relative_plug_index];
30545 if (plugs_to_fit != 0)
30551 if ((relative_plug_index > relative_free_space_index) ||
30552 ((relative_plug_index == relative_free_space_index) &&
30553 (plugs_to_fit > 1)))
30555 #ifdef SEG_REUSE_STATS
30556 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
30557 (relative_free_space_index + MIN_INDEX_POWER2),
30559 (relative_plug_index + MIN_INDEX_POWER2)));
30560 #endif // SEG_REUSE_STATS
30564 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
30565 ordered_free_space_indices[relative_free_space_index]++;
30566 use_bestfit = try_best_fit(TRUE);
30569 free_space_items++;
30570 // Since we might've trimmed away some of the free spaces we had, we should see
30571 // if we really need to use end of seg space - if it's the same or smaller than
30572 // the largest space we trimmed we can just add that one back instead of
30573 // using end of seg.
30574 if (relative_free_space_index > trimmed_free_space_index)
30576 *use_additional_space = TRUE;
30580 // If the addition space is <= than the last trimmed space, we
30581 // should just use that last trimmed space instead.
30582 saved_ordered_free_space_indices[trimmed_free_space_index]++;
30592 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
30594 #ifdef SEG_REUSE_STATS
30595 size_t saved_max = max_free_space_items;
30596 BOOL temp_bestfit = FALSE;
30598 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
30599 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
30601 // TODO: need to take the end of segment into consideration.
30602 while (max_free_space_items <= total_free_space_items)
30604 max_free_space_items += max_free_space_items / 2;
30605 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
30606 memcpy (ordered_free_space_indices,
30607 saved_all_free_space_indices,
30608 sizeof(ordered_free_space_indices));
30609 if (try_best_fit(FALSE))
30611 temp_bestfit = TRUE;
30618 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
30622 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
30625 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
30626 max_free_space_items = saved_max;
30627 #endif // SEG_REUSE_STATS
30628 if (free_space_items)
30630 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
30631 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
30635 max_free_space_items = MAX_NUM_FREE_SPACES;
30639 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
30640 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
30642 return use_bestfit;
30645 BOOL gc_heap::process_free_space (heap_segment* seg,
30647 size_t min_free_size,
30648 size_t min_cont_size,
30649 size_t* total_free_space,
30650 size_t* largest_free_space)
30652 *total_free_space += free_space;
30653 *largest_free_space = max (*largest_free_space, free_space);
30655 #ifdef SIMPLE_DPRINTF
30656 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
30657 free_space, *total_free_space, *largest_free_space));
30658 #endif //SIMPLE_DPRINTF
30660 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
30662 #ifdef SIMPLE_DPRINTF
30663 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
30664 settings.condemned_generation,
30665 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
30668 UNREFERENCED_PARAMETER(seg);
30669 #endif //SIMPLE_DPRINTF
30673 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
30674 if (free_space_index != -1)
30676 ordered_free_space_indices[free_space_index]++;
30681 BOOL gc_heap::expand_reused_seg_p()
30683 BOOL reused_seg = FALSE;
30684 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
30685 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
30686 (heap_expand_mechanism == expand_reuse_normal))
30694 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
30695 allocator* gen_allocator)
30697 min_cont_size += END_SPACE_AFTER_GC;
30698 use_bestfit = FALSE;
30699 commit_end_of_seg = FALSE;
30700 bestfit_first_pin = 0;
30701 uint8_t* first_address = heap_segment_mem (seg);
30702 uint8_t* end_address = heap_segment_reserved (seg);
30703 size_t end_extra_space = end_space_after_gc();
30705 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
30707 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
30708 first_address, end_address, end_extra_space));
30712 end_address -= end_extra_space;
30714 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
30715 settings.condemned_generation, min_free_size, min_cont_size));
30716 size_t eph_gen_starts = eph_gen_starts_size;
30718 if (settings.condemned_generation == max_generation)
30720 size_t free_space = 0;
30721 size_t largest_free_space = free_space;
30722 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
30723 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
30724 //We are going to allocate the generation starts in the 1st free space,
30725 //so start from the first free space that's big enough for gen starts and a min object size.
30726 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
30727 // we could use it by allocating the last generation start a bit bigger but
30728 // the complexity isn't worth the effort (those plugs are from gen2
30729 // already anyway).
30730 reset_pinned_queue_bos();
30732 BOOL has_fit_gen_starts = FALSE;
30734 init_ordered_free_space_indices ();
30735 while (!pinned_plug_que_empty_p())
30738 if ((pinned_plug (m) >= first_address) &&
30739 (pinned_plug (m) < end_address) &&
30740 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
30746 deque_pinned_plug();
30750 if (!pinned_plug_que_empty_p())
30752 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
30754 if (process_free_space (seg,
30755 pinned_len (m) - eph_gen_starts,
30756 min_free_size, min_cont_size,
30757 &free_space, &largest_free_space))
30762 deque_pinned_plug();
30764 has_fit_gen_starts = TRUE;
30767 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
30769 //tally up free space
30770 while (!pinned_plug_que_empty_p() &&
30771 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
30773 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
30774 if (process_free_space (seg,
30776 min_free_size, min_cont_size,
30777 &free_space, &largest_free_space))
30782 deque_pinned_plug();
30786 //try to find space at the end of the segment.
30787 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
30788 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
30789 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
30790 if (end_space >= additional_space)
30792 BOOL can_fit = TRUE;
30793 commit_end_of_seg = TRUE;
30795 if (largest_free_space < min_cont_size)
30797 if (end_space >= min_cont_size)
30799 additional_space = max (min_cont_size, additional_space);
30800 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
30805 if (settings.concurrent)
30808 commit_end_of_seg = FALSE;
30812 size_t additional_space_bestfit = additional_space;
30813 if (!has_fit_gen_starts)
30815 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
30817 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
30818 additional_space_bestfit));
30822 bestfit_first_pin = heap_segment_plan_allocated (seg);
30823 additional_space_bestfit -= eph_gen_starts;
30826 can_fit = best_fit (free_space,
30827 largest_free_space,
30828 additional_space_bestfit,
30829 &commit_end_of_seg);
30833 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
30834 seg, (commit_end_of_seg ? "with" : "without")));
30838 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
30845 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
30848 assert (additional_space <= end_space);
30849 if (commit_end_of_seg)
30851 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
30853 dprintf (2, ("Couldn't commit end of segment?!"));
30854 use_bestfit = FALSE;
30861 // We increase the index here because growing heap segment could create a discrepency with
30862 // the additional space we used (could be bigger).
30863 size_t free_space_end_of_seg =
30864 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
30865 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
30866 saved_ordered_free_space_indices[relative_free_space_index]++;
30872 memcpy (ordered_free_space_indices,
30873 saved_ordered_free_space_indices,
30874 sizeof(ordered_free_space_indices));
30875 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
30876 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
30877 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
30883 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
30888 assert (settings.condemned_generation == (max_generation-1));
30889 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
30890 size_t largest_free_space = free_space;
30891 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
30892 //find the first free list in range of the current segment
30893 uint8_t* free_list = 0;
30894 unsigned int a_l_idx = gen_allocator->first_suitable_bucket(eph_gen_starts);
30895 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
30897 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
30900 if ((free_list >= first_address) &&
30901 (free_list < end_address) &&
30902 (unused_array_size (free_list) >= eph_gen_starts))
30908 free_list = free_list_slot (free_list);
30915 init_ordered_free_space_indices ();
30916 if (process_free_space (seg,
30917 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
30918 min_free_size, min_cont_size,
30919 &free_space, &largest_free_space))
30924 free_list = free_list_slot (free_list);
30928 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
30932 //tally up free space
30937 if ((free_list >= first_address) && (free_list < end_address) &&
30938 process_free_space (seg,
30939 unused_array_size (free_list),
30940 min_free_size, min_cont_size,
30941 &free_space, &largest_free_space))
30946 free_list = free_list_slot (free_list);
30949 if (a_l_idx < gen_allocator->number_of_buckets())
30951 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
30957 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
30961 BOOL can_fit = best_fit (free_space, 0, NULL);
30964 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
30968 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
30976 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
30977 generation* gen, uint8_t* start_address,
30978 unsigned int& active_new_gen_number,
30979 uint8_t*& last_pinned_gap, BOOL& leftp,
30982 , mark* pinned_plug_entry
30983 #endif //SHORT_PLUGS
30986 // detect generation boundaries
30987 // make sure that active_new_gen_number is not the youngest generation.
30988 // because the generation_limit wouldn't return the right thing in this case.
30991 if ((active_new_gen_number > 1) &&
30992 (last_plug >= generation_limit (active_new_gen_number)))
30994 assert (last_plug >= start_address);
30995 active_new_gen_number--;
30996 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
30997 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
31002 // detect pinned plugs
31003 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
31005 size_t entry = deque_pinned_plug();
31006 mark* m = pinned_plug_of (entry);
31008 size_t saved_pinned_len = pinned_len(m);
31009 pinned_len(m) = last_plug - last_pinned_gap;
31010 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
31012 if (m->has_post_plug_info())
31014 last_plug_size += sizeof (gap_reloc_pair);
31015 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
31018 last_pinned_gap = last_plug + last_plug_size;
31019 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
31020 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
31023 //we are creating a generation fault. set the cards.
31025 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
31026 size_t card = card_of (last_plug);
31027 while (card != end_card)
31034 else if (last_plug >= start_address)
31036 #ifdef FEATURE_STRUCTALIGN
31037 int requiredAlignment;
31039 node_aligninfo (last_plug, requiredAlignment, pad);
31041 // from how we previously aligned the plug's destination address,
31042 // compute the actual alignment offset.
31043 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
31044 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
31045 if (!alignmentOffset)
31047 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
31048 alignmentOffset = requiredAlignment;
31051 //clear the alignment info because we are reallocating
31052 clear_node_aligninfo (last_plug);
31053 #else // FEATURE_STRUCTALIGN
31054 //clear the realignment flag because we are reallocating
31055 clear_node_realigned (last_plug);
31056 #endif // FEATURE_STRUCTALIGN
31057 BOOL adjacentp = FALSE;
31058 BOOL set_padding_on_saved_p = FALSE;
31062 last_plug_size += sizeof (gap_reloc_pair);
31065 assert (pinned_plug_entry != NULL);
31066 if (last_plug_size <= sizeof (plug_and_gap))
31068 set_padding_on_saved_p = TRUE;
31070 #endif //SHORT_PLUGS
31072 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
31076 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
31077 #endif //SHORT_PLUGS
31079 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
31081 set_padding_on_saved_p,
31083 #endif //SHORT_PLUGS
31084 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
31086 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
31087 assert (new_address);
31088 set_node_relocation_distance (last_plug, new_address - last_plug);
31089 #ifdef FEATURE_STRUCTALIGN
31090 if (leftp && node_alignpad (last_plug) == 0)
31091 #else // FEATURE_STRUCTALIGN
31092 if (leftp && !node_realigned (last_plug))
31093 #endif // FEATURE_STRUCTALIGN
31095 // TODO - temporarily disable L optimization because of a bug in it.
31096 //set_node_left (last_plug);
31098 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
31103 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
31104 uint8_t* start_address,
31106 unsigned int& active_new_gen_number,
31107 uint8_t*& last_pinned_gap, BOOL& leftp)
31109 assert (tree != NULL);
31110 int left_node = node_left_child (tree);
31111 int right_node = node_right_child (tree);
31113 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
31114 tree, last_pinned_gap, last_plug, left_node, right_node));
31118 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
31119 realloc_in_brick ((tree + left_node), last_plug, start_address,
31120 gen, active_new_gen_number, last_pinned_gap,
31124 if (last_plug != 0)
31126 uint8_t* plug = tree;
31128 BOOL has_pre_plug_info_p = FALSE;
31129 BOOL has_post_plug_info_p = FALSE;
31130 mark* pinned_plug_entry = get_next_pinned_entry (tree,
31131 &has_pre_plug_info_p,
31132 &has_post_plug_info_p,
31135 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
31136 // The pinned plugs are handled in realloc_plug.
31137 size_t gap_size = node_gap_size (plug);
31138 uint8_t* gap = (plug - gap_size);
31139 uint8_t* last_plug_end = gap;
31140 size_t last_plug_size = (last_plug_end - last_plug);
31141 // Cannot assert this - a plug could be less than that due to the shortened ones.
31142 //assert (last_plug_size >= Align (min_obj_size));
31143 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
31144 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
31145 realloc_plug (last_plug_size, last_plug, gen, start_address,
31146 active_new_gen_number, last_pinned_gap,
31147 leftp, has_pre_plug_info_p
31149 , pinned_plug_entry
31150 #endif //SHORT_PLUGS
31158 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
31159 realloc_in_brick ((tree + right_node), last_plug, start_address,
31160 gen, active_new_gen_number, last_pinned_gap,
31166 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
31167 uint8_t* start_address, uint8_t* end_address,
31168 unsigned active_new_gen_number)
31170 dprintf (3, ("--- Reallocing ---"));
31174 //make sure that every generation has a planned allocation start
31175 int gen_number = max_generation - 1;
31176 while (gen_number >= 0)
31178 generation* gen = generation_of (gen_number);
31179 if (0 == generation_plan_allocation_start (gen))
31181 generation_plan_allocation_start (gen) =
31182 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
31183 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
31184 assert (generation_plan_allocation_start (gen));
31190 uint8_t* first_address = start_address;
31191 //Look for the right pinned plug to start from.
31192 reset_pinned_queue_bos();
31193 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
31194 while (!pinned_plug_que_empty_p())
31196 mark* m = oldest_pin();
31197 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
31199 if (pinned_plug (m) < first_address)
31201 first_address = pinned_plug (m);
31206 deque_pinned_plug();
31209 size_t current_brick = brick_of (first_address);
31210 size_t end_brick = brick_of (end_address-1);
31211 uint8_t* last_plug = 0;
31213 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
31214 BOOL leftp = FALSE;
31216 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
31217 start_address, first_address, pinned_plug (oldest_pin())));
31219 while (current_brick <= end_brick)
31221 int brick_entry = brick_table [ current_brick ];
31222 if (brick_entry >= 0)
31224 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
31225 last_plug, start_address, consing_gen,
31226 active_new_gen_number, last_pinned_gap,
31232 if (last_plug != 0)
31234 realloc_plug (end_address - last_plug, last_plug, consing_gen,
31236 active_new_gen_number, last_pinned_gap,
31240 #endif //SHORT_PLUGS
31244 //Fix the old segment allocated size
31245 assert (last_pinned_gap >= heap_segment_mem (seg));
31246 assert (last_pinned_gap <= heap_segment_committed (seg));
31247 heap_segment_plan_allocated (seg) = last_pinned_gap;
31250 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
31253 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31255 BOOL contains_pinned_plugs = FALSE;
31258 while (mi != mark_stack_tos)
31260 m = pinned_plug_of (mi);
31261 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
31263 contains_pinned_plugs = TRUE;
31270 if (contains_pinned_plugs)
31275 #endif //VERIFY_HEAP
31278 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
31280 if (!should_expand_in_full_gc)
31282 if ((condemned_gen_number != max_generation) &&
31283 (settings.pause_mode != pause_low_latency) &&
31284 (settings.pause_mode != pause_sustained_low_latency))
31286 should_expand_in_full_gc = TRUE;
31291 void gc_heap::save_ephemeral_generation_starts()
31293 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
31295 saved_ephemeral_plan_start[ephemeral_generation] =
31296 generation_plan_allocation_start (generation_of (ephemeral_generation));
31297 saved_ephemeral_plan_start_size[ephemeral_generation] =
31298 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
31302 generation* gc_heap::expand_heap (int condemned_generation,
31303 generation* consing_gen,
31304 heap_segment* new_heap_segment)
31307 UNREFERENCED_PARAMETER(condemned_generation);
31309 assert (condemned_generation >= (max_generation -1));
31310 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
31311 uint8_t* start_address = generation_limit (max_generation);
31312 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
31313 BOOL should_promote_ephemeral = FALSE;
31314 ptrdiff_t eph_size = total_ephemeral_size;
31315 #ifdef BACKGROUND_GC
31316 dprintf(2,("%s: ---- Heap Expansion ----", (gc_heap::background_running_p() ? "FGC" : "NGC")));
31317 #endif //BACKGROUND_GC
31318 settings.heap_expansion = TRUE;
31320 #ifdef BACKGROUND_GC
31321 if (cm_in_progress)
31323 if (!expanded_in_fgc)
31325 expanded_in_fgc = TRUE;
31328 #endif //BACKGROUND_GC
31330 //reset the elevation state for next time.
31331 dprintf (2, ("Elevation: elevation = el_none"));
31332 if (settings.should_lock_elevation && !expand_reused_seg_p())
31333 settings.should_lock_elevation = FALSE;
31335 heap_segment* new_seg = new_heap_segment;
31338 return consing_gen;
31340 //copy the card and brick tables
31341 if (g_gc_card_table!= card_table)
31342 copy_brick_card_table();
31344 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
31345 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
31347 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
31348 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
31349 heap_segment_mem (ephemeral_heap_segment));
31350 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
31351 heap_segment_committed (ephemeral_heap_segment));
31353 assert (generation_plan_allocation_start (youngest_generation));
31354 assert (generation_plan_allocation_start (youngest_generation) <
31355 heap_segment_plan_allocated (ephemeral_heap_segment));
31357 if (settings.pause_mode == pause_no_gc)
31359 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
31360 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
31361 should_promote_ephemeral = TRUE;
31367 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
31371 if (should_promote_ephemeral)
31373 ephemeral_promotion = TRUE;
31374 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
31375 dprintf (2, ("promoting ephemeral"));
31376 save_ephemeral_generation_starts();
31380 // commit the new ephemeral segment all at once if it is a new one.
31381 if ((eph_size > 0) && new_segment_p)
31383 #ifdef FEATURE_STRUCTALIGN
31384 // The destination may require a larger alignment padding than the source.
31385 // Assume the worst possible alignment padding.
31386 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
31387 #endif // FEATURE_STRUCTALIGN
31388 #ifdef RESPECT_LARGE_ALIGNMENT
31389 //Since the generation start can be larger than min_obj_size
31390 //The alignment could be switched.
31391 eph_size += switch_alignment_size(FALSE);
31392 #endif //RESPECT_LARGE_ALIGNMENT
31393 //Since the generation start can be larger than min_obj_size
31394 //Compare the alignment of the first object in gen1
31395 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
31397 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
31398 return consing_gen;
31400 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
31403 //Fix the end of the old ephemeral heap segment
31404 heap_segment_plan_allocated (ephemeral_heap_segment) =
31405 generation_plan_allocation_start (generation_of (max_generation-1));
31407 dprintf (3, ("Old ephemeral allocated set to %Ix",
31408 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
31413 // TODO - Is this really necessary? We should think about it.
31414 //initialize the first brick
31415 size_t first_brick = brick_of (heap_segment_mem (new_seg));
31416 set_brick (first_brick,
31417 heap_segment_mem (new_seg) - brick_address (first_brick));
31420 //From this point on, we cannot run out of memory
31422 //reset the allocation of the consing generation back to the end of the
31423 //old ephemeral segment
31424 generation_allocation_limit (consing_gen) =
31425 heap_segment_plan_allocated (ephemeral_heap_segment);
31426 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
31427 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
31429 //clear the generation gap for all of the ephemeral generations
31431 int generation_num = max_generation-1;
31432 while (generation_num >= 0)
31434 generation* gen = generation_of (generation_num);
31435 generation_plan_allocation_start (gen) = 0;
31440 heap_segment* old_seg = ephemeral_heap_segment;
31441 ephemeral_heap_segment = new_seg;
31443 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
31444 //because the relocation and compact phases shouldn't see it
31446 // set the generation members used by allocate_in_expanded_heap
31447 // and switch to ephemeral generation
31448 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
31450 if (!should_promote_ephemeral)
31452 realloc_plugs (consing_gen, old_seg, start_address, end_address,
31453 active_new_gen_number);
31458 repair_allocation_in_expanded_heap (consing_gen);
31461 // assert that the generation gap for all of the ephemeral generations were allocated.
31464 int generation_num = max_generation-1;
31465 while (generation_num >= 0)
31467 generation* gen = generation_of (generation_num);
31468 assert (generation_plan_allocation_start (gen));
31474 if (!new_segment_p)
31476 dprintf (2, ("Demoting ephemeral segment"));
31477 //demote the entire segment.
31478 settings.demotion = TRUE;
31479 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
31480 demotion_low = heap_segment_mem (ephemeral_heap_segment);
31481 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
31485 demotion_low = MAX_PTR;
31487 #ifndef MULTIPLE_HEAPS
31488 settings.demotion = FALSE;
31489 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
31490 #endif //!MULTIPLE_HEAPS
31493 if (!should_promote_ephemeral && new_segment_p)
31495 assert ((ptrdiff_t)total_ephemeral_size <= eph_size);
31498 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
31500 // This is to catch when we accidently delete a segment that has pins.
31501 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
31504 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
31506 dprintf(2,("---- End of Heap Expansion ----"));
31507 return consing_gen;
31510 void gc_heap::set_static_data()
31512 static_data* pause_mode_sdata = static_data_table[latency_level];
31513 for (int i = 0; i < total_generation_count; i++)
31515 dynamic_data* dd = dynamic_data_of (i);
31516 static_data* sdata = &pause_mode_sdata[i];
31519 dd->min_size = sdata->min_size;
31521 dprintf (GTC_LOG, ("PM: %d, gen%d: min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
31522 settings.pause_mode,i,
31523 dd->min_size, dd_max_size (dd),
31524 sdata->fragmentation_limit, (int)(sdata->fragmentation_burden_limit * 100)));
31528 // Initialize the values that are not const.
31529 void gc_heap::init_static_data()
31531 size_t gen0_min_size = get_gen0_min_size();
31533 size_t gen0_max_size =
31534 #ifdef MULTIPLE_HEAPS
31535 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
31536 #else //MULTIPLE_HEAPS
31537 (gc_can_use_concurrent ?
31539 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
31540 #endif //MULTIPLE_HEAPS
31542 gen0_max_size = max (gen0_min_size, gen0_max_size);
31544 if (heap_hard_limit)
31546 size_t gen0_max_size_seg = soh_segment_size / 4;
31547 dprintf (GTC_LOG, ("limit gen0 max %Id->%Id", gen0_max_size, gen0_max_size_seg));
31548 gen0_max_size = min (gen0_max_size, gen0_max_size_seg);
31551 size_t gen0_max_size_config = (size_t)GCConfig::GetGCGen0MaxBudget();
31553 if (gen0_max_size_config)
31555 gen0_max_size = min (gen0_max_size, gen0_max_size_config);
31558 gen0_max_size = Align (gen0_max_size);
31559 gen0_min_size = min (gen0_min_size, gen0_max_size);
31561 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
31562 size_t gen1_max_size = (size_t)
31563 #ifdef MULTIPLE_HEAPS
31564 max (6*1024*1024, Align(soh_segment_size/2));
31565 #else //MULTIPLE_HEAPS
31566 (gc_can_use_concurrent ?
31568 max (6*1024*1024, Align(soh_segment_size/2)));
31569 #endif //MULTIPLE_HEAPS
31571 dprintf (GTC_LOG, ("gen0 min: %Id, max: %Id, gen1 max: %Id",
31572 gen0_min_size, gen0_max_size, gen1_max_size));
31574 for (int i = latency_level_first; i <= latency_level_last; i++)
31576 static_data_table[i][0].min_size = gen0_min_size;
31577 static_data_table[i][0].max_size = gen0_max_size;
31578 static_data_table[i][1].max_size = gen1_max_size;
31582 bool gc_heap::init_dynamic_data()
31584 uint64_t now_raw_ts = RawGetHighPrecisionTimeStamp ();
31585 #ifdef HEAP_BALANCE_INSTRUMENTATION
31586 start_raw_ts = now_raw_ts;
31587 #endif //HEAP_BALANCE_INSTRUMENTATION
31588 uint64_t now = (uint64_t)((double)now_raw_ts * qpf_us);
31592 if (heap_number == 0)
31594 process_start_time = now;
31595 smoothed_desired_per_heap = dynamic_data_of (0)->min_size;
31596 #ifdef HEAP_BALANCE_INSTRUMENTATION
31597 last_gc_end_time_us = now;
31598 dprintf (HEAP_BALANCE_LOG, ("qpf=%I64d, start: %I64d(%d)", qpf, start_raw_ts, now));
31599 #endif //HEAP_BALANCE_INSTRUMENTATION
31602 for (int i = 0; i < total_generation_count; i++)
31604 dynamic_data* dd = dynamic_data_of (i);
31606 dd->time_clock = now;
31607 dd->current_size = 0;
31608 dd->promoted_size = 0;
31609 dd->collection_count = 0;
31610 dd->new_allocation = dd->min_size;
31611 dd->gc_new_allocation = dd->new_allocation;
31612 dd->desired_allocation = dd->new_allocation;
31613 dd->fragmentation = 0;
31619 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
31621 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
31622 return ((limit - limit*cst) / (1.0f - (cst * limit)));
31628 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
31629 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
31630 //value of the budget
31631 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
31632 size_t previous_desired_allocation, size_t collection_count)
31634 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
31636 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
31637 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
31640 size_t smoothing = 3; // exponential smoothing factor
31641 if (smoothing > collection_count)
31642 smoothing = collection_count;
31643 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
31645 UNREFERENCED_PARAMETER(collection_count);
31647 return new_allocation;
31650 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
31651 size_t out, int gen_number,
31654 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
31656 if (dd_begin_data_size (dd) == 0)
31658 size_t new_allocation = dd_min_size (dd);
31659 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
31660 return new_allocation;
31665 size_t previous_desired_allocation = dd_desired_allocation (dd);
31666 size_t current_size = dd_current_size (dd);
31667 float max_limit = dd_max_limit (dd);
31668 float limit = dd_limit (dd);
31669 size_t min_gc_size = dd_min_size (dd);
31671 size_t max_size = dd_max_size (dd);
31672 size_t new_allocation = 0;
31673 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
31675 if (gen_number >= max_generation)
31677 size_t new_size = 0;
31679 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
31681 f = surv_to_growth (cst, limit, max_limit);
31682 size_t max_growth_size = (size_t)(max_size / f);
31683 if (current_size >= max_growth_size)
31685 new_size = max_size;
31689 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
31692 assert ((new_size >= current_size) || (new_size == max_size));
31694 if (gen_number == max_generation)
31696 new_allocation = max((new_size - current_size), min_gc_size);
31698 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
31699 dd_desired_allocation (dd), dd_collection_count (dd));
31702 #ifdef BGC_SERVO_TUNING
31703 !bgc_tuning::fl_tuning_triggered &&
31704 #endif //BGC_SERVO_TUNING
31705 (dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
31707 //reducing allocation in case of fragmentation
31708 size_t new_allocation1 = max (min_gc_size,
31710 (size_t)((float)new_allocation * current_size /
31711 ((float)current_size + 2*dd_fragmentation (dd))));
31712 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
31713 new_allocation, new_allocation1));
31714 new_allocation = new_allocation1;
31717 else // not a SOH generation
31719 uint32_t memory_load = 0;
31720 uint64_t available_physical = 0;
31721 get_memory_info (&memory_load, &available_physical);
31723 if (heap_hard_limit)
31725 size_t allocated = 0;
31726 size_t committed = uoh_committed_size (gen_number, &allocated);
31727 dprintf (1, ("GC#%Id h%d, GMI: UOH budget, UOH commit %Id (obj %Id, frag %Id), total commit: %Id (recorded: %Id)",
31728 (size_t)settings.gc_index, heap_number,
31729 committed, allocated,
31730 dd_fragmentation (dynamic_data_of (gen_number)),
31731 get_total_committed_size(), (current_total_committed - current_total_committed_bookkeeping)));
31734 if (heap_number == 0)
31735 settings.exit_memory_load = memory_load;
31736 if (available_physical > 1024*1024)
31737 available_physical -= 1024*1024;
31739 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
31740 if (available_free > (uint64_t)MAX_PTR)
31742 available_free = (uint64_t)MAX_PTR;
31745 //try to avoid OOM during large object allocation
31746 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
31747 (size_t)available_free),
31748 max ((current_size/4), min_gc_size));
31750 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
31751 dd_desired_allocation (dd), dd_collection_count (dd));
31757 size_t survivors = out;
31758 cst = float (survivors) / float (dd_begin_data_size (dd));
31759 f = surv_to_growth (cst, limit, max_limit);
31760 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
31762 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
31763 dd_desired_allocation (dd), dd_collection_count (dd));
31765 if (gen_number == 0)
31770 //printf ("%f, %Id\n", cst, new_allocation);
31771 size_t free_space = generation_free_list_space (generation_of (gen_number));
31772 // DTREVIEW - is min_gc_size really a good choice?
31773 // on 64-bit this will almost always be true.
31774 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
31775 if (free_space > min_gc_size)
31777 settings.gen0_reduction_count = 2;
31781 if (settings.gen0_reduction_count > 0)
31782 settings.gen0_reduction_count--;
31785 if (settings.gen0_reduction_count > 0)
31787 dprintf (2, ("Reducing new allocation based on fragmentation"));
31788 new_allocation = min (new_allocation,
31789 max (min_gc_size, (max_size/3)));
31794 size_t new_allocation_ret = Align (new_allocation, get_alignment_constant (gen_number <= max_generation));
31795 int gen_data_index = gen_number;
31796 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
31797 gen_data->new_allocation = new_allocation_ret;
31799 dd_surv (dd) = cst;
31801 #ifdef SIMPLE_DPRINTF
31802 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
31803 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
31804 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
31806 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
31807 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
31808 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
31809 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
31810 #endif //SIMPLE_DPRINTF
31812 return new_allocation_ret;
31816 //returns the planned size of a generation (including free list element)
31817 size_t gc_heap::generation_plan_size (int gen_number)
31819 if (0 == gen_number)
31820 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
31821 generation_plan_allocation_start (generation_of (gen_number))),
31822 (int)Align (min_obj_size));
31825 generation* gen = generation_of (gen_number);
31826 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
31827 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
31828 generation_plan_allocation_start (generation_of (gen_number)));
31831 size_t gensize = 0;
31832 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31834 PREFIX_ASSUME(seg != NULL);
31836 while (seg && (seg != ephemeral_heap_segment))
31838 gensize += heap_segment_plan_allocated (seg) -
31839 heap_segment_mem (seg);
31840 seg = heap_segment_next_rw (seg);
31844 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
31845 heap_segment_mem (ephemeral_heap_segment));
31853 //returns the size of a generation (including free list element)
31854 size_t gc_heap::generation_size (int gen_number)
31856 if (0 == gen_number)
31857 return max((heap_segment_allocated (ephemeral_heap_segment) -
31858 generation_allocation_start (generation_of (gen_number))),
31859 (int)Align (min_obj_size));
31862 generation* gen = generation_of (gen_number);
31863 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
31864 return (generation_allocation_start (generation_of (gen_number - 1)) -
31865 generation_allocation_start (generation_of (gen_number)));
31868 size_t gensize = 0;
31869 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31871 PREFIX_ASSUME(seg != NULL);
31873 while (seg && (seg != ephemeral_heap_segment))
31875 gensize += heap_segment_allocated (seg) -
31876 heap_segment_mem (seg);
31877 seg = heap_segment_next_rw (seg);
31881 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
31882 heap_segment_mem (ephemeral_heap_segment));
31891 size_t gc_heap::compute_in (int gen_number)
31893 assert (gen_number != 0);
31894 dynamic_data* dd = dynamic_data_of (gen_number);
31896 size_t in = generation_allocation_size (generation_of (gen_number));
31898 if (gen_number == max_generation && ephemeral_promotion)
31901 for (int i = 0; i <= max_generation; i++)
31903 dynamic_data* dd = dynamic_data_of (i);
31904 in += dd_survived_size (dd);
31905 if (i != max_generation)
31907 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
31912 dd_gc_new_allocation (dd) -= in;
31913 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
31915 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
31916 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
31919 generation_allocation_size (generation_of (gen_number)) = 0;
31923 void gc_heap::compute_promoted_allocation (int gen_number)
31925 compute_in (gen_number);
31930 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
31931 size_t total_new_allocation,
31932 size_t total_min_allocation)
31934 if (memory_load < MAX_ALLOWED_MEM_LOAD)
31936 // If the total of memory load and gen0 budget exceeds
31937 // our max memory load limit, trim the gen0 budget so the total
31938 // is the max memory load limit.
31939 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
31940 return min (total_new_allocation, remain_memory_load);
31944 return max (mem_one_percent, total_min_allocation);
31948 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
31950 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
31952 size_t final_new_allocation = new_allocation;
31953 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
31955 uint32_t num_heaps = 1;
31957 #ifdef MULTIPLE_HEAPS
31958 num_heaps = gc_heap::n_heaps;
31959 #endif //MULTIPLE_HEAPS
31961 size_t total_new_allocation = new_allocation * num_heaps;
31962 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
31964 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
31965 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
31967 uint32_t memory_load = 0;
31968 get_memory_info (&memory_load);
31969 settings.exit_memory_load = memory_load;
31970 dprintf (2, ("Current memory load: %d", memory_load));
31972 size_t final_total =
31973 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
31974 size_t max_new_allocation =
31975 #ifdef MULTIPLE_HEAPS
31976 dd_max_size (g_heaps[0]->dynamic_data_of (0));
31977 #else //MULTIPLE_HEAPS
31978 dd_max_size (dynamic_data_of (0));
31979 #endif //MULTIPLE_HEAPS
31981 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
31985 if (final_new_allocation < new_allocation)
31987 settings.gen0_reduction_count = 2;
31990 return final_new_allocation;
31992 #endif // HOST_64BIT
31995 gc_history_global* gc_heap::get_gc_data_global()
31997 #ifdef BACKGROUND_GC
31998 return (settings.concurrent ? &bgc_data_global : &gc_data_global);
32000 return &gc_data_global;
32001 #endif //BACKGROUND_GC
32005 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
32007 #ifdef BACKGROUND_GC
32008 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
32010 return &gc_data_per_heap;
32011 #endif //BACKGROUND_GC
32014 void gc_heap::compute_new_dynamic_data (int gen_number)
32016 PREFIX_ASSUME(gen_number >= 0);
32017 PREFIX_ASSUME(gen_number <= max_generation);
32019 dynamic_data* dd = dynamic_data_of (gen_number);
32020 generation* gen = generation_of (gen_number);
32021 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
32023 size_t total_gen_size = generation_size (gen_number);
32024 //keep track of fragmentation
32025 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
32026 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
32028 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
32030 size_t out = dd_survived_size (dd);
32032 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
32033 gen_data->size_after = total_gen_size;
32034 gen_data->free_list_space_after = generation_free_list_space (gen);
32035 gen_data->free_obj_space_after = generation_free_obj_space (gen);
32037 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
32039 // When we are in the low latency mode, we can still be
32040 // condemning more than gen1's 'cause of induced GCs.
32041 dd_desired_allocation (dd) = low_latency_alloc;
32045 if (gen_number == 0)
32047 //compensate for dead finalizable objects promotion.
32048 //they shoudn't be counted for growth.
32049 size_t final_promoted = 0;
32050 final_promoted = min (promoted_bytes (heap_number), out);
32051 // Prefast: this is clear from above but prefast needs to be told explicitly
32052 PREFIX_ASSUME(final_promoted <= out);
32054 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
32055 dd_freach_previous_promotion (dd) = final_promoted;
32056 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
32058 if (settings.condemned_generation == 0)
32060 //there is no noise.
32061 dd_desired_allocation (dd) = lower_bound;
32065 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
32067 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
32068 //assert ( lower_bound <= higher_bound);
32070 //discount the noise. Change the desired allocation
32071 //only if the previous value is outside of the range.
32072 if (dd_desired_allocation (dd) < lower_bound)
32074 dd_desired_allocation (dd) = lower_bound;
32076 else if (dd_desired_allocation (dd) > higher_bound)
32078 dd_desired_allocation (dd) = higher_bound;
32080 #if defined (HOST_64BIT) && !defined (MULTIPLE_HEAPS)
32081 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
32082 #endif // HOST_64BIT && !MULTIPLE_HEAPS
32083 trim_youngest_desired_low_memory();
32084 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
32089 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
32093 gen_data->pinned_surv = dd_pinned_survived_size (dd);
32094 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
32096 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
32097 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
32099 dd_promoted_size (dd) = out;
32100 if (gen_number == max_generation)
32102 for (int i = (gen_number + 1); i < total_generation_count; i++)
32104 dd = dynamic_data_of (i);
32105 total_gen_size = generation_size (i);
32106 generation* gen = generation_of (i);
32107 dd_fragmentation (dd) = generation_free_list_space (gen) +
32108 generation_free_obj_space (gen);
32109 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
32110 dd_survived_size (dd) = dd_current_size (dd);
32112 out = dd_current_size (dd);
32113 dd_desired_allocation (dd) = desired_new_allocation (dd, out, i, 0);
32114 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
32115 get_alignment_constant (FALSE));
32116 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
32118 gen_data = &(current_gc_data_per_heap->gen_data[i]);
32119 gen_data->size_after = total_gen_size;
32120 gen_data->free_list_space_after = generation_free_list_space (gen);
32121 gen_data->free_obj_space_after = generation_free_obj_space (gen);
32122 gen_data->npinned_surv = out;
32123 #ifdef BACKGROUND_GC
32124 if (i == loh_generation)
32125 end_loh_size = total_gen_size;
32127 if (i == poh_generation)
32128 end_poh_size = total_gen_size;
32129 #endif //BACKGROUND_GC
32130 dd_promoted_size (dd) = out;
32135 void gc_heap::trim_youngest_desired_low_memory()
32137 if (g_low_memory_status)
32139 size_t committed_mem = committed_size();
32140 dynamic_data* dd = dynamic_data_of (0);
32141 size_t current = dd_desired_allocation (dd);
32142 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
32144 dd_desired_allocation (dd) = min (current, candidate);
32148 void gc_heap::decommit_ephemeral_segment_pages()
32150 if (settings.concurrent || use_large_pages_p)
32155 dynamic_data* dd0 = dynamic_data_of (0);
32157 // this is how much we are going to allocate in gen 0
32158 ptrdiff_t desired_allocation = dd_desired_allocation (dd0) + loh_size_threshold;
32160 // estimate how we are going to need in gen 1 - estimate half the free list space gets used
32161 dynamic_data* dd1 = dynamic_data_of (1);
32162 ptrdiff_t desired_allocation_1 = dd_new_allocation (dd1) - (generation_free_list_space (generation_of (1)) / 2);
32163 if (desired_allocation_1 > 0)
32165 desired_allocation += desired_allocation_1;
32168 size_t slack_space =
32170 max(min(min(soh_segment_size/32, dd_max_size (dd0)), (generation_size (max_generation) / 10)), (size_t)desired_allocation);
32172 #ifdef FEATURE_CORECLR
32173 desired_allocation;
32176 #endif //FEATURE_CORECLR
32177 #endif // HOST_64BIT
32179 uint8_t *decommit_target = heap_segment_allocated (ephemeral_heap_segment) + slack_space;
32180 if (decommit_target < heap_segment_decommit_target (ephemeral_heap_segment))
32182 // we used to have a higher target - do exponential smoothing by computing
32183 // essentially decommit_target = 1/3*decommit_target + 2/3*previous_decommit_target
32184 // computation below is slightly different to avoid overflow
32185 ptrdiff_t target_decrease = heap_segment_decommit_target (ephemeral_heap_segment) - decommit_target;
32186 decommit_target += target_decrease * 2 / 3;
32189 heap_segment_decommit_target(ephemeral_heap_segment) = decommit_target;
32191 #ifdef MULTIPLE_HEAPS
32192 if (decommit_target < heap_segment_committed (ephemeral_heap_segment))
32194 gradual_decommit_in_progress_p = TRUE;
32197 // these are only for checking against logic errors
32198 ephemeral_heap_segment->saved_committed = heap_segment_committed (ephemeral_heap_segment);
32199 ephemeral_heap_segment->saved_desired_allocation = dd_desired_allocation (dd0);
32201 #endif // MULTIPLE_HEAPS
32203 #ifndef MULTIPLE_HEAPS
32204 // we want to limit the amount of decommit we do per time to indirectly
32205 // limit the amount of time spent in recommit and page faults
32206 size_t ephemeral_elapsed = (size_t)((dd_time_clock (dd0) - gc_last_ephemeral_decommit_time) / 1000);
32207 gc_last_ephemeral_decommit_time = dd_time_clock (dd0);
32209 // this is the amount we were planning to decommit
32210 ptrdiff_t decommit_size = heap_segment_committed (ephemeral_heap_segment) - decommit_target;
32212 // we do a max of DECOMMIT_SIZE_PER_MILLISECOND per millisecond of elapsed time since the last GC
32213 // we limit the elapsed time to 10 seconds to avoid spending too much time decommitting
32214 ptrdiff_t max_decommit_size = min (ephemeral_elapsed, (10*1000)) * DECOMMIT_SIZE_PER_MILLISECOND;
32215 decommit_size = min (decommit_size, max_decommit_size);
32217 slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment) - decommit_size;
32218 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
32219 #endif // !MULTIPLE_HEAPS
32221 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
32222 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
32225 #ifdef MULTIPLE_HEAPS
32226 // return true if we actually decommitted anything
32227 bool gc_heap::decommit_step ()
32229 // should never get here for large pages because decommit_ephemeral_segment_pages
32230 // will not do anything if use_large_pages_p is true
32231 assert (!use_large_pages_p);
32233 size_t decommit_size = 0;
32234 for (int i = 0; i < n_heaps; i++)
32236 gc_heap* hp = gc_heap::g_heaps[i];
32237 decommit_size += hp->decommit_ephemeral_segment_pages_step ();
32239 return (decommit_size != 0);
32242 // return the decommitted size
32243 size_t gc_heap::decommit_ephemeral_segment_pages_step ()
32245 // we rely on desired allocation not being changed outside of GC
32246 assert (ephemeral_heap_segment->saved_desired_allocation == dd_desired_allocation (dynamic_data_of (0)));
32248 uint8_t* decommit_target = heap_segment_decommit_target (ephemeral_heap_segment);
32249 size_t EXTRA_SPACE = 2 * OS_PAGE_SIZE;
32250 decommit_target += EXTRA_SPACE;
32251 uint8_t* committed = heap_segment_committed (ephemeral_heap_segment);
32252 if (decommit_target < committed)
32254 // we rely on other threads not messing with committed if we are about to trim it down
32255 assert (ephemeral_heap_segment->saved_committed == heap_segment_committed (ephemeral_heap_segment));
32257 // how much would we need to decommit to get to decommit_target in one step?
32258 size_t full_decommit_size = (committed - decommit_target);
32260 // don't do more than max_decommit_step_size per step
32261 size_t decommit_size = min (max_decommit_step_size, full_decommit_size);
32263 // figure out where the new committed should be
32264 uint8_t* new_committed = (committed - decommit_size);
32265 size_t size = decommit_heap_segment_pages_worker (ephemeral_heap_segment, new_committed);
32268 ephemeral_heap_segment->saved_committed = committed - size;
32275 #endif //MULTIPLE_HEAPS
32277 //This is meant to be called by decide_on_compacting.
32279 size_t gc_heap::generation_fragmentation (generation* gen,
32280 generation* consing_gen,
32284 uint8_t* alloc = generation_allocation_pointer (consing_gen);
32285 // If the allocation pointer has reached the ephemeral segment
32286 // fine, otherwise the whole ephemeral segment is considered
32288 if (in_range_for_segment (alloc, ephemeral_heap_segment))
32290 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
32291 frag = end - alloc;
32294 // case when no survivors, allocated set to beginning
32297 dprintf (3, ("ephemeral frag: %Id", frag));
32300 frag = (heap_segment_allocated (ephemeral_heap_segment) -
32301 heap_segment_mem (ephemeral_heap_segment));
32302 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32304 PREFIX_ASSUME(seg != NULL);
32306 while (seg != ephemeral_heap_segment)
32308 frag += (heap_segment_allocated (seg) -
32309 heap_segment_plan_allocated (seg));
32310 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
32311 (heap_segment_allocated (seg) -
32312 heap_segment_plan_allocated (seg))));
32314 seg = heap_segment_next_rw (seg);
32317 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
32318 //add the length of the dequeued plug free space
32320 while (bos < mark_stack_bos)
32322 frag += (pinned_len (pinned_plug_of (bos)));
32329 // for SOH this returns the total sizes of the generation and its
32330 // younger generation(s).
32331 // for LOH this returns just LOH size.
32332 size_t gc_heap::generation_sizes (generation* gen)
32335 if (generation_start_segment (gen ) == ephemeral_heap_segment)
32336 result = (heap_segment_allocated (ephemeral_heap_segment) -
32337 generation_allocation_start (gen));
32340 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
32342 PREFIX_ASSUME(seg != NULL);
32346 result += (heap_segment_allocated (seg) -
32347 heap_segment_mem (seg));
32348 seg = heap_segment_next_in_range (seg);
32355 size_t gc_heap::estimated_reclaim (int gen_number)
32357 dynamic_data* dd = dynamic_data_of (gen_number);
32358 size_t gen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
32359 size_t gen_total_size = gen_allocated + dd_current_size (dd);
32360 size_t est_gen_surv = (size_t)((float) (gen_total_size) * dd_surv (dd));
32361 size_t est_gen_free = gen_total_size - est_gen_surv + dd_fragmentation (dd);
32363 dprintf (GTC_LOG, ("h%d gen%d total size: %Id, est dead space: %Id (s: %d, allocated: %Id), frag: %Id",
32364 heap_number, gen_number,
32367 (int)(dd_surv (dd) * 100),
32369 dd_fragmentation (dd)));
32371 return est_gen_free;
32374 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
32375 size_t fragmentation,
32376 BOOL& should_expand)
32378 BOOL should_compact = FALSE;
32379 should_expand = FALSE;
32380 generation* gen = generation_of (condemned_gen_number);
32381 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
32382 size_t gen_sizes = generation_sizes(gen);
32383 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
32384 (float (fragmentation) / gen_sizes) );
32386 dprintf (GTC_LOG, ("h%d g%d fragmentation: %Id (%d%%)",
32387 heap_number, settings.condemned_generation,
32388 fragmentation, (int)(fragmentation_burden * 100.0)));
32390 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
32391 // for GC stress runs we need compaction
32392 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent)
32393 should_compact = TRUE;
32394 #endif //defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
32396 if (GCConfig::GetForceCompact())
32397 should_compact = TRUE;
32399 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
32401 should_compact = TRUE;
32402 last_gc_before_oom = FALSE;
32403 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
32406 if (settings.reason == reason_induced_compacting)
32408 dprintf (2, ("induced compacting GC"));
32409 should_compact = TRUE;
32410 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
32413 if (settings.reason == reason_pm_full_gc)
32415 assert (condemned_gen_number == max_generation);
32416 if (heap_number == 0)
32418 dprintf (GTC_LOG, ("PM doing compacting full GC after a gen1"));
32420 should_compact = TRUE;
32423 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
32424 fragmentation, (int) (100*fragmentation_burden)));
32426 if (provisional_mode_triggered && (condemned_gen_number == (max_generation - 1)))
32428 dprintf (GTC_LOG, ("gen1 in PM always compact"));
32429 should_compact = TRUE;
32432 if (!should_compact)
32434 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
32436 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
32437 should_compact = TRUE;
32438 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
32442 if (should_compact)
32444 if ((condemned_gen_number >= (max_generation - 1)))
32446 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
32448 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
32449 should_expand = TRUE;
32455 BOOL high_memory = FALSE;
32456 #endif // HOST_64BIT
32458 if (!should_compact)
32460 // We are not putting this in dt_high_frag_p because it's not exactly
32461 // high fragmentation - it's just enough planned fragmentation for us to
32462 // want to compact. Also the "fragmentation" we are talking about here
32463 // is different from anywhere else.
32464 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
32465 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
32469 #ifdef BACKGROUND_GC
32470 // do not force compaction if this was a stress-induced GC
32471 IN_STRESS_HEAP(if (!settings.stress_induced))
32473 #endif // BACKGROUND_GC
32474 assert (settings.concurrent == FALSE);
32475 should_compact = TRUE;
32476 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
32477 #ifdef BACKGROUND_GC
32479 #endif // BACKGROUND_GC
32483 // check for high memory situation
32484 if(!should_compact)
32486 uint32_t num_heaps = 1;
32487 #ifdef MULTIPLE_HEAPS
32488 num_heaps = gc_heap::n_heaps;
32489 #endif // MULTIPLE_HEAPS
32491 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
32493 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
32495 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
32497 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
32498 should_compact = TRUE;
32499 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
32501 high_memory = TRUE;
32503 else if(settings.entry_memory_load >= v_high_memory_load_th)
32505 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
32507 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
32508 should_compact = TRUE;
32509 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
32511 high_memory = TRUE;
32514 #endif // HOST_64BIT
32517 // The purpose of calling ensure_gap_allocation here is to make sure
32518 // that we actually are able to commit the memory to allocate generation
32520 if ((should_compact == FALSE) &&
32521 (ensure_gap_allocation (condemned_gen_number) == FALSE))
32523 should_compact = TRUE;
32524 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
32527 if (settings.condemned_generation == max_generation)
32529 //check the progress
32532 (high_memory && !should_compact) ||
32533 #endif // HOST_64BIT
32534 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
32535 generation_allocation_start (generation_of (max_generation - 1))))
32537 dprintf (1, ("gen1 start %Ix->%Ix, gen2 size %Id->%Id, lock elevation",
32538 generation_allocation_start (generation_of (max_generation - 1)),
32539 generation_plan_allocation_start (generation_of (max_generation - 1)),
32540 generation_size (max_generation),
32541 generation_plan_size (max_generation)));
32542 //no progress -> lock
32543 settings.should_lock_elevation = TRUE;
32547 if (settings.pause_mode == pause_no_gc)
32549 should_compact = TRUE;
32550 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
32551 < soh_allocation_no_gc)
32553 should_expand = TRUE;
32557 dprintf (2, ("will %s(%s)", (should_compact ? "compact" : "sweep"), (should_expand ? "ex" : "")));
32558 return should_compact;
32561 size_t align_lower_good_size_allocation (size_t size)
32563 return (size/64)*64;
32566 size_t gc_heap::approximate_new_allocation()
32568 dynamic_data* dd0 = dynamic_data_of (0);
32569 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
32572 BOOL gc_heap::sufficient_space_end_seg (uint8_t* start, uint8_t* seg_end, size_t end_space_required, gc_tuning_point tp)
32574 BOOL can_fit = FALSE;
32575 size_t end_seg_space = (size_t)(seg_end - start);
32576 if (end_seg_space > end_space_required)
32578 // If hard limit is specified, and if we attributed all that's left in commit to the ephemeral seg
32579 // so we treat that as segment end, do we have enough space.
32580 if (heap_hard_limit)
32582 size_t left_in_commit = heap_hard_limit - current_total_committed;
32584 #ifdef MULTIPLE_HEAPS
32585 num_heaps = n_heaps;
32586 #endif //MULTIPLE_HEAPS
32587 left_in_commit /= num_heaps;
32588 if (left_in_commit > end_space_required)
32593 dprintf (2, ("h%d end seg %Id, but only %Id left in HARD LIMIT commit, required: %Id %s on eph (%d)",
32594 heap_number, end_seg_space,
32595 left_in_commit, end_space_required,
32596 (can_fit ? "ok" : "short"), (int)tp));
32605 // After we did a GC we expect to have at least this
32606 // much space at the end of the segment to satisfy
32607 // a reasonable amount of allocation requests.
32608 size_t gc_heap::end_space_after_gc()
32610 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
32613 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
32615 uint8_t* start = 0;
32617 if ((tp == tuning_deciding_condemned_gen) ||
32618 (tp == tuning_deciding_compaction))
32620 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
32621 if (settings.concurrent)
32623 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
32624 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
32628 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
32629 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
32632 else if (tp == tuning_deciding_expansion)
32634 start = heap_segment_plan_allocated (ephemeral_heap_segment);
32635 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
32636 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
32640 assert (tp == tuning_deciding_full_gc);
32641 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
32642 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
32643 start = alloc_allocated;
32646 if (start == 0) // empty ephemeral generations
32648 assert (tp == tuning_deciding_expansion);
32649 // if there are no survivors in the ephemeral segment,
32650 // this should be the beginning of ephemeral segment.
32651 start = generation_allocation_pointer (generation_of (max_generation));
32652 assert (start == heap_segment_mem (ephemeral_heap_segment));
32655 if (tp == tuning_deciding_expansion)
32657 assert (settings.condemned_generation >= (max_generation-1));
32658 size_t gen0size = approximate_new_allocation();
32659 size_t eph_size = gen0size;
32660 size_t gen_min_sizes = 0;
32662 for (int j = 1; j <= max_generation-1; j++)
32664 gen_min_sizes += 2*dd_min_size (dynamic_data_of(j));
32667 eph_size += gen_min_sizes;
32669 dprintf (3, ("h%d deciding on expansion, need %Id (gen0: %Id, 2*min: %Id)",
32670 heap_number, gen0size, gen_min_sizes, eph_size));
32672 // We must find room for one large object and enough room for gen0size
32673 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
32675 dprintf (3, ("Enough room before end of segment"));
32680 size_t room = align_lower_good_size_allocation
32681 (heap_segment_reserved (ephemeral_heap_segment) - start);
32682 size_t end_seg = room;
32684 //look at the plug free space
32685 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
32686 bool large_chunk_found = FALSE;
32688 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
32689 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
32690 if (gen0start == 0)
32692 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
32694 while ((bos < mark_stack_bos) &&
32695 !((room >= gen0size) && large_chunk_found))
32697 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
32698 if (in_range_for_segment (plug, ephemeral_heap_segment))
32700 if (plug >= gen0start)
32702 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
32704 if (!large_chunk_found)
32706 large_chunk_found = (chunk >= largest_alloc);
32708 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
32709 room, large_chunk_found));
32715 if (room >= gen0size)
32717 if (large_chunk_found)
32719 sufficient_gen0_space_p = TRUE;
32721 dprintf (3, ("Enough room"));
32726 // now we need to find largest_alloc at the end of the segment.
32727 if (end_seg >= end_space_after_gc())
32729 dprintf (3, ("Enough room (may need end of seg)"));
32735 dprintf (3, ("Not enough room"));
32741 size_t end_space = 0;
32742 dynamic_data* dd = dynamic_data_of (0);
32743 if ((tp == tuning_deciding_condemned_gen) ||
32744 (tp == tuning_deciding_full_gc))
32746 end_space = max (2*dd_min_size (dd), end_space_after_gc());
32750 assert (tp == tuning_deciding_compaction);
32751 end_space = approximate_new_allocation();
32754 BOOL can_fit = sufficient_space_end_seg (start, heap_segment_reserved (ephemeral_heap_segment), end_space, tp);
32760 CObjectHeader* gc_heap::allocate_uoh_object (size_t jsize, uint32_t flags, int gen_number, int64_t& alloc_bytes)
32762 //create a new alloc context because gen3context is shared.
32763 alloc_context acontext;
32767 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
32769 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
32772 if (jsize >= maxObjectSize)
32774 if (GCConfig::GetBreakOnOOM())
32776 GCToOSInterface::DebugBreak();
32781 size_t size = AlignQword (jsize);
32782 int align_const = get_alignment_constant (FALSE);
32784 #ifdef FEATURE_LOH_COMPACTION
32785 if (gen_number == loh_generation)
32787 pad = Align (loh_padding_obj_size, align_const);
32789 #endif //FEATURE_LOH_COMPACTION
32791 assert (size >= Align (min_obj_size, align_const));
32793 #pragma inline_depth(0)
32795 if (! allocate_more_space (&acontext, (size + pad), flags, gen_number))
32801 #pragma inline_depth(20)
32804 #ifdef FEATURE_LOH_COMPACTION
32805 // The GC allocator made a free object already in this alloc context and
32806 // adjusted the alloc_ptr accordingly.
32807 #endif //FEATURE_LOH_COMPACTION
32809 uint8_t* result = acontext.alloc_ptr;
32811 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
32812 alloc_bytes += size;
32814 CObjectHeader* obj = (CObjectHeader*)result;
32816 #ifdef BACKGROUND_GC
32817 if (gc_heap::background_running_p())
32819 uint8_t* current_lowest_address = background_saved_lowest_address;
32820 uint8_t* current_highest_address = background_saved_highest_address;
32822 if ((result < current_highest_address) && (result >= current_lowest_address))
32824 dprintf (3, ("Clearing mark bit at address %Ix",
32825 (size_t)(&mark_array [mark_word_of (result)])));
32827 mark_array_clear_marked (result);
32829 if (current_c_gc_state != c_gc_state_free)
32831 dprintf (3, ("Concurrent allocation of a large object %Ix",
32833 //mark the new block specially so we know it is a new object
32834 if ((result < current_highest_address) && (result >= current_lowest_address))
32836 dprintf (3, ("Setting mark bit at address %Ix",
32837 (size_t)(&mark_array [mark_word_of (result)])));
32839 mark_array_set_marked (result);
32843 #endif //BACKGROUND_GC
32846 assert ((size_t)obj == Align ((size_t)obj, align_const));
32851 void reset_memory (uint8_t* o, size_t sizeo)
32853 if (sizeo > 128 * 1024)
32855 // We cannot reset the memory for the useful part of a free object.
32856 size_t size_to_skip = min_free_list - plug_skew;
32858 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
32859 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
32860 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
32861 // on write watched memory.
32862 if (reset_mm_p && gc_heap::g_low_memory_status)
32864 #ifdef MULTIPLE_HEAPS
32865 bool unlock_p = true;
32867 // We don't do unlock because there could be many processes using workstation GC and it's
32868 // bad perf to have many threads doing unlock at the same time.
32869 bool unlock_p = false;
32870 #endif //MULTIPLE_HEAPS
32872 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
32877 BOOL gc_heap::uoh_object_marked (uint8_t* o, BOOL clearp)
32880 // It shouldn't be necessary to do these comparisons because this is only used for blocking
32881 // GCs and LOH segments cannot be out of range.
32882 if ((o >= lowest_address) && (o < highest_address))
32902 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
32904 // Now walk the portion of memory that is actually being relocated.
32905 walk_relocation (profiling_context, fn);
32907 #ifdef FEATURE_LOH_COMPACTION
32908 if (loh_compacted_p)
32910 walk_relocation_for_loh (profiling_context, fn);
32912 #endif //FEATURE_LOH_COMPACTION
32915 void gc_heap::walk_survivors_for_uoh (void* profiling_context, record_surv_fn fn, int gen_number)
32917 generation* gen = generation_of (gen_number);
32918 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
32920 PREFIX_ASSUME(seg != NULL);
32922 uint8_t* o = generation_allocation_start (gen);
32923 uint8_t* plug_end = o;
32924 uint8_t* plug_start = o;
32928 if (o >= heap_segment_allocated (seg))
32930 seg = heap_segment_next (seg);
32934 o = heap_segment_mem (seg);
32936 if (uoh_object_marked(o, FALSE))
32943 o = o + AlignQword (size (o));
32944 if (o >= heap_segment_allocated (seg))
32948 m = uoh_object_marked (o, FALSE);
32953 fn (plug_start, plug_end, 0, profiling_context, false, false);
32957 while (o < heap_segment_allocated (seg) && !uoh_object_marked(o, FALSE))
32959 o = o + AlignQword (size (o));
32965 #ifdef BACKGROUND_GC
32967 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
32970 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
32972 if (mark_array_marked (o))
32976 mark_array_clear_marked (o);
32977 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
32978 dprintf (3, ("CM: %Ix", o));
32988 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
32992 void gc_heap::background_delay_delete_uoh_segments()
32994 for (int i = uoh_start_generation; i < total_generation_count; i++)
32996 generation* gen = generation_of (i);
32997 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32998 heap_segment* prev_seg = 0;
33002 heap_segment* next_seg = heap_segment_next (seg);
33003 if (seg->flags & heap_segment_flags_uoh_delete)
33005 dprintf (3, ("deleting %Ix-%Ix-%Ix", (size_t)seg, heap_segment_allocated (seg), heap_segment_reserved (seg)));
33006 delete_heap_segment (seg, (GCConfig::GetRetainVM() != 0));
33007 heap_segment_next (prev_seg) = next_seg;
33019 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
33022 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
33025 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
33030 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
33031 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
33033 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
33034 memset (start, b, (end - start));
33037 #endif //VERIFY_HEAP
33040 void gc_heap::generation_delete_heap_segment (generation* gen,
33042 heap_segment* prev_seg,
33043 heap_segment* next_seg)
33045 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
33046 if (gen->gen_num > max_generation)
33048 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
33050 // We cannot thread segs in here onto freeable_uoh_segment because
33051 // grow_brick_card_tables could be committing mark array which needs to read
33052 // the seg list. So we delay it till next time we suspend EE.
33053 seg->flags |= heap_segment_flags_uoh_delete;
33054 // Since we will be decommitting the seg, we need to prevent heap verification
33055 // to verify this segment.
33056 heap_segment_allocated (seg) = heap_segment_mem (seg);
33060 if (seg == ephemeral_heap_segment)
33065 heap_segment_next (next_seg) = prev_seg;
33067 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
33068 heap_segment_next (seg) = freeable_soh_segment;
33069 freeable_soh_segment = seg;
33072 decommit_heap_segment (seg);
33073 seg->flags |= heap_segment_flags_decommitted;
33075 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
33078 void gc_heap::process_background_segment_end (heap_segment* seg,
33080 uint8_t* last_plug_end,
33081 heap_segment* start_seg,
33085 uint8_t* allocated = heap_segment_allocated (seg);
33086 uint8_t* background_allocated = heap_segment_background_allocated (seg);
33087 BOOL uoh_p = heap_segment_uoh_p (seg);
33089 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
33090 (size_t)heap_segment_mem (seg), background_allocated, allocated));
33092 if (!uoh_p && (allocated != background_allocated))
33094 assert (gen->gen_num <= max_generation);
33096 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
33097 (size_t)last_plug_end, background_allocated));
33098 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
33101 fix_brick_to_highest (last_plug_end, background_allocated);
33103 // When we allowed fgc's during going through gaps, we could have erased the brick
33104 // that corresponds to bgc_allocated 'cause we had to update the brick there,
33105 // recover it here.
33106 fix_brick_to_highest (background_allocated, background_allocated);
33110 // by default, if allocated == background_allocated, it can't
33111 // be the ephemeral segment.
33112 if (seg == ephemeral_heap_segment)
33117 if (allocated == heap_segment_mem (seg))
33119 // this can happen with UOH segments when multiple threads
33120 // allocate new segments and not all of them were needed to
33121 // satisfy allocation requests.
33122 assert (gen->gen_num > max_generation);
33125 if (last_plug_end == heap_segment_mem (seg))
33127 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
33128 (size_t)allocated, (*delete_p ? "should" : "should not")));
33130 if (seg != start_seg)
33137 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
33138 heap_segment_allocated (seg) = last_plug_end;
33139 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
33141 decommit_heap_segment_pages (seg, 0);
33145 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
33146 bgc_verify_mark_array_cleared (seg);
33150 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
33152 BOOL consider_bgc_mark_p,
33153 BOOL check_current_sweep_p,
33154 BOOL check_saved_sweep_p)
33156 // the logic for this function must be kept in sync with the analogous function
33157 // in ToolBox\SOS\Strike\gc.cpp
33159 // TRUE means we don't need to check the bgc mark bit
33160 // FALSE means we do.
33161 BOOL no_bgc_mark_p = FALSE;
33163 if (consider_bgc_mark_p)
33165 if (check_current_sweep_p && (o < current_sweep_pos))
33167 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
33168 no_bgc_mark_p = TRUE;
33171 if (!no_bgc_mark_p)
33173 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
33175 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
33176 no_bgc_mark_p = TRUE;
33179 if (!check_saved_sweep_p)
33181 uint8_t* background_allocated = heap_segment_background_allocated (seg);
33182 // if this was the saved ephemeral segment, check_saved_sweep_p
33183 // would've been true.
33184 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
33185 // background_allocated could be 0 for the new segments acquired during bgc
33186 // sweep and we still want no_bgc_mark_p to be true.
33187 if (o >= background_allocated)
33189 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
33190 no_bgc_mark_p = TRUE;
33197 no_bgc_mark_p = TRUE;
33200 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
33201 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
33204 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
33205 // if it's TRUE, check_current_sweep_p tells you if you should consider the
33206 // current sweep position or not.
33207 void gc_heap::should_check_bgc_mark (heap_segment* seg,
33208 BOOL* consider_bgc_mark_p,
33209 BOOL* check_current_sweep_p,
33210 BOOL* check_saved_sweep_p)
33212 // the logic for this function must be kept in sync with the analogous function
33213 // in ToolBox\SOS\Strike\gc.cpp
33214 *consider_bgc_mark_p = FALSE;
33215 *check_current_sweep_p = FALSE;
33216 *check_saved_sweep_p = FALSE;
33218 if (current_c_gc_state == c_gc_state_planning)
33220 // We are doing the current_sweep_pos comparison here because we have yet to
33221 // turn on the swept flag for the segment but in_range_for_segment will return
33222 // FALSE if the address is the same as reserved.
33223 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
33225 dprintf (3, ("seg %Ix is already swept by bgc", seg));
33229 *consider_bgc_mark_p = TRUE;
33231 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
33233 if (seg == saved_sweep_ephemeral_seg)
33235 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
33236 *check_saved_sweep_p = TRUE;
33239 if (in_range_for_segment (current_sweep_pos, seg))
33241 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
33242 current_sweep_pos, seg));
33243 *check_current_sweep_p = TRUE;
33249 void gc_heap::background_ephemeral_sweep()
33251 dprintf (3, ("bgc ephemeral sweep"));
33253 int align_const = get_alignment_constant (TRUE);
33255 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
33256 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
33258 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
33259 // we thread onto a list first then publish it when we are done.
33260 allocator youngest_free_list;
33261 size_t youngest_free_list_space = 0;
33262 size_t youngest_free_obj_space = 0;
33264 youngest_free_list.clear();
33266 for (int i = 0; i <= (max_generation - 1); i++)
33268 generation* gen_to_reset = generation_of (i);
33269 assert (generation_free_list_space (gen_to_reset) == 0);
33270 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
33271 // something there.
33274 for (int i = (max_generation - 1); i >= 0; i--)
33276 generation* current_gen = generation_of (i);
33277 uint8_t* o = generation_allocation_start (current_gen);
33278 //Skip the generation gap object
33279 o = o + Align(size (o), align_const);
33280 uint8_t* end = ((i > 0) ?
33281 generation_allocation_start (generation_of (i - 1)) :
33282 heap_segment_allocated (ephemeral_heap_segment));
33284 uint8_t* plug_end = o;
33285 uint8_t* plug_start = o;
33286 BOOL marked_p = FALSE;
33290 marked_p = background_object_marked (o, TRUE);
33294 size_t plug_size = plug_start - plug_end;
33298 thread_gap (plug_end, plug_size, current_gen);
33304 make_unused_array (plug_end, plug_size);
33305 if (plug_size >= min_free_list)
33307 youngest_free_list_space += plug_size;
33308 youngest_free_list.thread_item (plug_end, plug_size);
33312 youngest_free_obj_space += plug_size;
33317 fix_brick_to_highest (plug_end, plug_start);
33318 fix_brick_to_highest (plug_start, plug_start);
33323 o = o + Align (size (o), align_const);
33329 m = background_object_marked (o, TRUE);
33332 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
33336 while ((o < end) && !background_object_marked (o, FALSE))
33338 o = o + Align (size (o), align_const);
33343 if (plug_end != end)
33347 thread_gap (plug_end, end - plug_end, current_gen);
33351 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
33352 // the following line is temporary.
33353 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
33354 make_unused_array (plug_end, (end - plug_end));
33357 fix_brick_to_highest (plug_end, end);
33360 dd_fragmentation (dynamic_data_of (i)) =
33361 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
33364 generation* youngest_gen = generation_of (0);
33365 generation_free_list_space (youngest_gen) = youngest_free_list_space;
33366 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
33367 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
33368 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
33371 void gc_heap::background_sweep()
33373 //concurrent_print_time_delta ("finished with mark and start with sweep");
33374 concurrent_print_time_delta ("Sw");
33375 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
33377 //block concurrent allocation for large objects
33378 dprintf (3, ("lh state: planning"));
33380 for (int i = 0; i <= max_generation; i++)
33382 generation* gen_to_reset = generation_of (i);
33383 generation_allocator (gen_to_reset)->clear();
33384 generation_free_list_space (gen_to_reset) = 0;
33385 generation_free_obj_space (gen_to_reset) = 0;
33386 generation_free_list_allocated (gen_to_reset) = 0;
33387 generation_end_seg_allocated (gen_to_reset) = 0;
33388 generation_condemned_allocated (gen_to_reset) = 0;
33389 generation_sweep_allocated (gen_to_reset) = 0;
33390 //reset the allocation so foreground gc can allocate into older generation
33391 generation_allocation_pointer (gen_to_reset)= 0;
33392 generation_allocation_limit (gen_to_reset) = 0;
33393 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
33396 FIRE_EVENT(BGC2ndNonConEnd);
33398 uoh_alloc_thread_count = 0;
33399 current_bgc_state = bgc_sweep_soh;
33400 verify_soh_segment_list();
33402 #ifdef FEATURE_BASICFREEZE
33403 generation* max_gen = generation_of (max_generation);
33404 if ((generation_start_segment (max_gen) != ephemeral_heap_segment) &&
33405 ro_segments_in_range)
33407 sweep_ro_segments (generation_start_segment (max_gen));
33409 #endif // FEATURE_BASICFREEZE
33411 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
33412 if (current_c_gc_state != c_gc_state_planning)
33414 current_c_gc_state = c_gc_state_planning;
33417 concurrent_print_time_delta ("Swe");
33419 for (int i = uoh_start_generation; i < total_generation_count; i++)
33421 heap_segment* uoh_seg = heap_segment_rw (generation_start_segment (generation_of (i)));
33422 PREFIX_ASSUME(uoh_seg != NULL);
33425 uoh_seg->flags &= ~heap_segment_flags_swept;
33426 heap_segment_background_allocated (uoh_seg) = heap_segment_allocated (uoh_seg);
33427 uoh_seg = heap_segment_next_rw (uoh_seg);
33431 #ifdef MULTIPLE_HEAPS
33432 bgc_t_join.join(this, gc_join_restart_ee);
33433 if (bgc_t_join.joined())
33435 dprintf(2, ("Starting BGC threads for resuming EE"));
33436 bgc_t_join.restart();
33438 #endif //MULTIPLE_HEAPS
33440 if (heap_number == 0)
33442 #ifdef BGC_SERVO_TUNING
33443 get_and_reset_loh_alloc_info();
33444 #endif //BGC_SERVO_TUNING
33445 uint64_t suspended_end_ts = GetHighPrecisionTimeStamp();
33446 last_bgc_info[last_bgc_info_index].pause_durations[1] = (size_t)(suspended_end_ts - suspended_start_time);
33447 total_suspended_time += last_bgc_info[last_bgc_info_index].pause_durations[1];
33451 FIRE_EVENT(BGC2ndConBegin);
33453 background_ephemeral_sweep();
33455 concurrent_print_time_delta ("Swe eph");
33457 #ifdef MULTIPLE_HEAPS
33458 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
33459 if (bgc_t_join.joined())
33460 #endif //MULTIPLE_HEAPS
33462 #ifdef FEATURE_EVENT_TRACE
33463 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
33464 GCEventKeyword_GCHeapSurvivalAndMovement,
33465 GCEventLevel_Information);
33466 #endif //FEATURE_EVENT_TRACE
33468 leave_spin_lock (&gc_lock);
33470 #ifdef MULTIPLE_HEAPS
33471 dprintf(2, ("Starting BGC threads for BGC sweeping"));
33472 bgc_t_join.restart();
33473 #endif //MULTIPLE_HEAPS
33476 disable_preemptive (true);
33478 dynamic_data* dd = dynamic_data_of (max_generation);
33479 const int num_objs = 256;
33480 int current_num_objs = 0;
33482 for (int i = max_generation; i < total_generation_count; i++)
33484 generation* gen = generation_of (i);
33485 heap_segment* gen_start_seg = heap_segment_rw (generation_start_segment(gen));
33486 heap_segment* next_seg = 0;
33487 heap_segment* prev_seg;
33488 heap_segment* start_seg;
33491 if (i == max_generation)
33493 // start with saved ephemeral segment
33494 // we are no longer holding gc_lock, so a new ephemeral segment could be added, we want the saved one.
33495 start_seg = saved_sweep_ephemeral_seg;
33496 prev_seg = heap_segment_next(start_seg);
33497 align_const = get_alignment_constant (TRUE);
33501 start_seg = gen_start_seg;
33503 align_const = get_alignment_constant (FALSE);
33505 // UOH allocations are possible while sweeping SOH, so
33506 // we defer clearing UOH free lists until we start sweeping them
33507 generation_allocator (gen)->clear();
33508 generation_free_list_space (gen) = 0;
33509 generation_free_obj_space (gen) = 0;
33510 generation_free_list_allocated (gen) = 0;
33511 generation_end_seg_allocated (gen) = 0;
33512 generation_condemned_allocated (gen) = 0;
33513 generation_sweep_allocated (gen) = 0;
33514 generation_allocation_pointer (gen)= 0;
33515 generation_allocation_limit (gen) = 0;
33516 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
33519 PREFIX_ASSUME(start_seg != NULL);
33520 heap_segment* seg = start_seg;
33521 dprintf (2, ("bgs: sweeping gen %Ix objects", gen->gen_num));
33524 uint8_t* o = heap_segment_mem (seg);
33525 if (seg == gen_start_seg)
33527 assert (o == generation_allocation_start (gen));
33528 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
33529 o = o + Align (size (o), align_const);
33532 uint8_t* plug_end = o;
33533 current_sweep_pos = o;
33534 next_sweep_obj = o;
33537 uint8_t* end = background_next_end (seg, (i > max_generation));
33538 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
33539 (size_t)heap_segment_mem (seg),
33540 (size_t)heap_segment_allocated (seg),
33541 (size_t)heap_segment_background_allocated (seg)));
33545 if (background_object_marked (o, TRUE))
33547 uint8_t* plug_start = o;
33548 if (i > max_generation)
33550 dprintf (2, ("uoh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
33553 thread_gap (plug_end, plug_start-plug_end, gen);
33554 if (i == max_generation)
33556 add_gen_free (max_generation, plug_start-plug_end);
33557 fix_brick_to_highest (plug_end, plug_start);
33558 // we need to fix the brick for the next plug here 'cause an FGC can
33559 // happen and can't read a stale brick.
33560 fix_brick_to_highest (plug_start, plug_start);
33565 o = o + Align (size (o), align_const);
33566 current_num_objs++;
33567 if (current_num_objs >= num_objs)
33569 current_sweep_pos = o;
33571 current_num_objs = 0;
33573 } while ((o < end) && background_object_marked(o, TRUE));
33576 if (i == max_generation)
33578 add_gen_plug (max_generation, plug_end-plug_start);
33579 dd_survived_size (dd) += (plug_end - plug_start);
33581 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
33584 while ((o < end) && !background_object_marked (o, FALSE))
33586 o = o + Align (size (o), align_const);;
33587 current_num_objs++;
33588 if (current_num_objs >= num_objs)
33590 current_sweep_pos = plug_end;
33591 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
33593 current_num_objs = 0;
33598 if (i > max_generation)
33600 next_seg = heap_segment_next (seg);
33604 // For SOH segments we go backwards.
33605 next_seg = heap_segment_prev (gen_start_seg, seg);
33608 BOOL delete_p = FALSE;
33609 if (!heap_segment_read_only_p (seg))
33611 if (i > max_generation)
33613 // we can treat all UOH segments as in the bgc domain
33614 // regardless of whether we saw in bgc mark or not
33615 // because we don't allow UOH allocations during bgc
33616 // sweep anyway - the UOH segments can't change.
33617 process_background_segment_end (seg, gen, plug_end,
33618 start_seg, &delete_p);
33622 assert (heap_segment_background_allocated (seg) != 0);
33623 process_background_segment_end (seg, gen, plug_end,
33624 start_seg, &delete_p);
33626 assert (next_seg || !delete_p);
33632 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
33637 dprintf (2, ("seg %Ix has been swept", seg));
33638 seg->flags |= heap_segment_flags_swept;
33641 verify_soh_segment_list();
33644 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
33647 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
33648 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
33650 if (i == max_generation)
33652 dprintf (2, ("bgs: sweeping uoh objects"));
33653 concurrent_print_time_delta ("Swe SOH");
33654 FIRE_EVENT(BGC1stSweepEnd, 0);
33656 enter_spin_lock (&more_space_lock_uoh);
33657 add_saved_spinlock_info (true, me_acquire, mt_bgc_uoh_sweep);
33659 concurrent_print_time_delta ("Swe UOH took msl");
33661 // We wait till all allocating threads are completely done.
33662 int spin_count = yp_spin_count_unit;
33663 while (uoh_alloc_thread_count)
33665 spin_and_switch (spin_count, (uoh_alloc_thread_count == 0));
33668 current_bgc_state = bgc_sweep_uoh;
33672 size_t total_soh_size = generation_sizes (generation_of (max_generation));
33673 size_t total_loh_size = generation_size (loh_generation);
33674 size_t total_poh_size = generation_size (poh_generation);
33676 dprintf (GTC_LOG, ("h%d: S: poh: %Id, loh: %Id, soh: %Id", heap_number, total_poh_size, total_loh_size, total_soh_size));
33678 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
33679 generation_free_list_space (generation_of (max_generation)),
33680 generation_free_obj_space (generation_of (max_generation))));
33682 dprintf (GTC_LOG, ("h%d: end of bgc sweep: loh FL: %Id, FO: %Id",
33684 generation_free_list_space (generation_of (loh_generation)),
33685 generation_free_obj_space (generation_of (loh_generation))));
33687 dprintf (GTC_LOG, ("h%d: end of bgc sweep: poh FL: %Id, FO: %Id",
33689 generation_free_list_space (generation_of (poh_generation)),
33690 generation_free_obj_space (generation_of (poh_generation))));
33692 FIRE_EVENT(BGC2ndConEnd);
33693 concurrent_print_time_delta ("background sweep");
33695 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
33696 PREFIX_ASSUME(reset_seg != NULL);
33700 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
33701 heap_segment_background_allocated (reset_seg) = 0;
33702 reset_seg = heap_segment_next_rw (reset_seg);
33705 // We calculate dynamic data here because if we wait till we signal the lh event,
33706 // the allocation thread can change the fragmentation and we may read an intermediate
33707 // value (which can be greater than the generation size). Plus by that time it won't
33709 compute_new_dynamic_data (max_generation);
33711 enable_preemptive ();
33713 #ifdef MULTIPLE_HEAPS
33714 bgc_t_join.join(this, gc_join_set_state_free);
33715 if (bgc_t_join.joined())
33716 #endif //MULTIPLE_HEAPS
33718 // TODO: We are using this join just to set the state. Should
33719 // look into eliminating it - check to make sure things that use
33720 // this state can live with per heap state like should_check_bgc_mark.
33721 current_c_gc_state = c_gc_state_free;
33723 #ifdef BGC_SERVO_TUNING
33724 if (bgc_tuning::enable_fl_tuning)
33726 enter_spin_lock (&gc_lock);
33727 bgc_tuning::record_and_adjust_bgc_end();
33728 leave_spin_lock (&gc_lock);
33730 #endif //BGC_SERVO_TUNING
33732 #ifdef MULTIPLE_HEAPS
33733 dprintf(2, ("Starting BGC threads after background sweep phase"));
33734 bgc_t_join.restart();
33735 #endif //MULTIPLE_HEAPS
33738 disable_preemptive (true);
33740 add_saved_spinlock_info (true, me_release, mt_bgc_uoh_sweep);
33741 leave_spin_lock (&more_space_lock_uoh);
33743 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
33744 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
33746 #endif //BACKGROUND_GC
33748 void gc_heap::sweep_uoh_objects (int gen_num)
33750 //this min value is for the sake of the dynamic tuning.
33751 //so we know that we are not starting even if we have no
33753 generation* gen = generation_of (gen_num);
33754 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
33756 PREFIX_ASSUME(start_seg != NULL);
33758 heap_segment* seg = start_seg;
33759 heap_segment* prev_seg = 0;
33760 uint8_t* o = generation_allocation_start (gen);
33761 int align_const = get_alignment_constant (FALSE);
33763 //Skip the generation gap object
33764 o = o + Align(size (o), align_const);
33766 uint8_t* plug_end = o;
33767 uint8_t* plug_start = o;
33769 generation_allocator (gen)->clear();
33770 generation_free_list_space (gen) = 0;
33771 generation_free_obj_space (gen) = 0;
33774 dprintf (3, ("sweeping uoh objects"));
33775 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
33777 (size_t)heap_segment_mem (seg),
33778 (size_t)heap_segment_allocated (seg),
33783 if (o >= heap_segment_allocated (seg))
33785 heap_segment* next_seg = heap_segment_next (seg);
33786 //delete the empty segment if not the only one
33787 if ((plug_end == heap_segment_mem (seg)) &&
33788 (seg != start_seg) && !heap_segment_read_only_p (seg))
33790 //prepare for deletion
33791 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
33793 heap_segment_next (prev_seg) = next_seg;
33794 heap_segment_next (seg) = freeable_uoh_segment;
33795 freeable_uoh_segment = seg;
33799 if (!heap_segment_read_only_p (seg))
33801 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
33802 heap_segment_allocated (seg) = plug_end;
33803 decommit_heap_segment_pages (seg, 0);
33812 o = heap_segment_mem (seg);
33814 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
33815 (size_t)heap_segment_mem (seg),
33816 (size_t)heap_segment_allocated (seg)));
33819 if (uoh_object_marked(o, TRUE))
33822 //everything between plug_end and plug_start is free
33823 thread_gap (plug_end, plug_start-plug_end, gen);
33828 o = o + AlignQword (size (o));
33829 if (o >= heap_segment_allocated (seg))
33833 m = uoh_object_marked (o, TRUE);
33836 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
33840 while (o < heap_segment_allocated (seg) && !uoh_object_marked(o, FALSE))
33842 o = o + AlignQword (size (o));
33847 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
33849 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
33852 void gc_heap::relocate_in_uoh_objects (int gen_num)
33854 generation* gen = generation_of (gen_num);
33856 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33858 PREFIX_ASSUME(seg != NULL);
33860 uint8_t* o = generation_allocation_start (gen);
33864 if (o >= heap_segment_allocated (seg))
33866 seg = heap_segment_next_rw (seg);
33871 o = heap_segment_mem (seg);
33874 while (o < heap_segment_allocated (seg))
33876 check_class_object_demotion (o);
33877 if (contain_pointers (o))
33879 dprintf(3, ("Relocating through uoh object %Ix", (size_t)o));
33880 go_through_object_nostart (method_table (o), o, size(o), pval,
33882 reloc_survivor_helper (pval);
33885 o = o + AlignQword (size (o));
33890 void gc_heap::mark_through_cards_for_uoh_objects (card_fn fn,
33893 CARD_MARKING_STEALING_ARG(gc_heap* hpt))
33895 uint8_t* low = gc_low;
33896 size_t end_card = 0;
33897 generation* oldest_gen = generation_of (gen_num);
33898 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
33900 PREFIX_ASSUME(seg != NULL);
33902 uint8_t* beg = generation_allocation_start (oldest_gen);
33903 uint8_t* end = heap_segment_allocated (seg);
33905 size_t cg_pointers_found = 0;
33907 size_t card_word_end = (card_of (align_on_card_word (end)) /
33912 size_t n_card_set = 0;
33913 uint8_t* next_boundary = (relocating ?
33914 generation_plan_allocation_start (generation_of (max_generation -1)) :
33917 uint8_t* nhigh = (relocating ?
33918 heap_segment_plan_allocated (ephemeral_heap_segment) :
33921 BOOL foundp = FALSE;
33922 uint8_t* start_address = 0;
33923 uint8_t* limit = 0;
33924 size_t card = card_of (beg);
33926 #ifdef BACKGROUND_GC
33927 BOOL consider_bgc_mark_p = FALSE;
33928 BOOL check_current_sweep_p = FALSE;
33929 BOOL check_saved_sweep_p = FALSE;
33930 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33931 #endif //BACKGROUND_GC
33933 size_t total_cards_cleared = 0;
33935 #ifdef FEATURE_CARD_MARKING_STEALING
33936 VOLATILE(uint32_t)* chunk_index = (VOLATILE(uint32_t)*) &(gen_num == loh_generation ?
33937 card_mark_chunk_index_loh :
33938 card_mark_chunk_index_poh);
33940 card_marking_enumerator card_mark_enumerator(seg, low, chunk_index);
33942 #endif // FEATURE_CARD_MARKING_STEALING
33944 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
33945 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
33948 if ((o < end) && (card_of(o) > card))
33950 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
33951 if (cg_pointers_found == 0)
33953 uint8_t* last_object_processed = o;
33954 #ifdef FEATURE_CARD_MARKING_STEALING
33955 last_object_processed = min(limit, o);
33956 #endif // FEATURE_CARD_MARKING_STEALING
33957 dprintf (3, (" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object_processed));
33958 clear_cards (card, card_of((uint8_t*)last_object_processed));
33959 total_cards_cleared += (card_of((uint8_t*)last_object_processed) - card);
33961 n_eph +=cg_pointers_found;
33962 cg_pointers_found = 0;
33963 card = card_of ((uint8_t*)o);
33965 if ((o < end) &&(card >= end_card))
33967 #ifdef FEATURE_CARD_MARKING_STEALING
33968 // find another chunk with some cards set
33969 foundp = find_next_chunk(card_mark_enumerator, seg, n_card_set, start_address, limit, card, end_card, card_word_end);
33970 #else // FEATURE_CARD_MARKING_STEALING
33971 foundp = find_card (card_table, card, card_word_end, end_card);
33974 n_card_set+= end_card - card;
33975 start_address = max (beg, card_address (card));
33977 limit = min (end, card_address (end_card));
33978 #endif // FEATURE_CARD_MARKING_STEALING
33980 if ((!foundp) || (o >= end) || (card_address (card) >= end))
33982 if ((foundp) && (cg_pointers_found == 0))
33984 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
33985 (size_t)card_address(card+1)));
33986 clear_cards (card, card+1);
33987 total_cards_cleared += 1;
33989 n_eph +=cg_pointers_found;
33990 cg_pointers_found = 0;
33991 #ifdef FEATURE_CARD_MARKING_STEALING
33992 // we have decided to move to the next segment - make sure we exhaust the chunk enumerator for this segment
33993 card_mark_enumerator.exhaust_segment(seg);
33994 #endif // FEATURE_CARD_MARKING_STEALING
33995 if ((seg = heap_segment_next_rw (seg)) != 0)
33997 #ifdef BACKGROUND_GC
33998 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33999 #endif //BACKGROUND_GC
34000 beg = heap_segment_mem (seg);
34001 end = compute_next_end (seg, low);
34002 #ifdef FEATURE_CARD_MARKING_STEALING
34004 #else // FEATURE_CARD_MARKING_STEALING
34005 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
34006 #endif // FEATURE_CARD_MARKING_STEALING
34007 card = card_of (beg);
34018 assert (card_set_p (card));
34020 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
34021 card, (size_t)o, (size_t)limit));
34023 assert (Align (size (o)) >= Align (min_obj_size));
34024 size_t s = size (o);
34025 uint8_t* next_o = o + AlignQword (s);
34031 assert (Align (s) >= Align (min_obj_size));
34032 next_o = o + AlignQword (s);
34035 dprintf (4, ("|%Ix|", (size_t)o));
34036 if (next_o < start_address)
34041 #ifdef BACKGROUND_GC
34042 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
34046 #endif //BACKGROUND_GC
34048 #ifdef COLLECTIBLE_CLASS
34049 if (is_collectible(o))
34051 BOOL passed_end_card_p = FALSE;
34053 if (card_of (o) > card)
34055 passed_end_card_p = card_transition (o, end, card_word_end,
34059 foundp, start_address,
34060 limit, total_cards_cleared
34061 CARD_MARKING_STEALING_ARGS(card_mark_enumerator, seg, card_word_end));
34064 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
34066 // card is valid and it covers the head of the object
34067 if (fn == &gc_heap::relocate_address)
34069 keep_card_live (o, n_gen, cg_pointers_found);
34073 uint8_t* class_obj = get_class_object (o);
34074 mark_through_cards_helper (&class_obj, n_gen,
34075 cg_pointers_found, fn,
34076 nhigh, next_boundary CARD_MARKING_STEALING_ARG(hpt));
34080 if (passed_end_card_p)
34082 if (foundp && (card_address (card) < next_o))
34084 goto go_through_refs;
34094 #endif //COLLECTIBLE_CLASS
34096 if (contain_pointers (o))
34098 dprintf(3,("Going through %Ix", (size_t)o));
34100 go_through_object (method_table(o), o, s, poo,
34101 start_address, use_start, (o + s),
34103 if (card_of ((uint8_t*)poo) > card)
34105 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
34110 foundp, start_address,
34111 limit, total_cards_cleared
34112 CARD_MARKING_STEALING_ARGS(card_mark_enumerator, seg, card_word_end));
34114 if (passed_end_card_p)
34116 if (foundp && (card_address (card) < next_o))
34120 if (ppstop <= (uint8_t**)start_address)
34122 else if (poo < (uint8_t**)start_address)
34123 {poo = (uint8_t**)start_address;}
34133 mark_through_cards_helper (poo, n_gen,
34134 cg_pointers_found, fn,
34135 nhigh, next_boundary CARD_MARKING_STEALING_ARG(hpt));
34147 // compute the efficiency ratio of the card table
34150 generation_skip_ratio = min (((n_eph > 800) ?
34151 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
34152 generation_skip_ratio);
34154 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
34155 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
34159 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
34160 n_eph, n_gen, n_card_set, generation_skip_ratio));
34164 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
34166 #ifdef MULTIPLE_HEAPS
34167 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
34168 for (int i = 0; i < n_heaps; i++)
34170 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
34171 #else //MULTIPLE_HEAPS
34173 gc_heap* hp = NULL;
34175 // prefix complains about us dereferencing hp in wks build even though we only access static members
34176 // this way. not sure how to shut it up except for this ugly workaround:
34177 PREFIX_ASSUME(hp != NULL);
34178 #endif // _PREFAST_
34179 #endif //MULTIPLE_HEAPS
34181 for (int curr_gen_number = total_generation_count-1; curr_gen_number >= 0; curr_gen_number--)
34183 generation* gen = hp->generation_of (curr_gen_number);
34184 heap_segment* seg = generation_start_segment (gen);
34185 while (seg && (seg != hp->ephemeral_heap_segment))
34187 assert (curr_gen_number > 0);
34189 // report bounds from heap_segment_mem (seg) to
34190 // heap_segment_allocated (seg);
34191 // for generation # curr_gen_number
34192 // for heap # heap_no
34194 fn(context, curr_gen_number, heap_segment_mem (seg),
34195 heap_segment_allocated (seg),
34196 curr_gen_number > max_generation ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
34198 seg = heap_segment_next (seg);
34203 assert (seg == hp->ephemeral_heap_segment);
34204 assert (curr_gen_number <= max_generation);
34206 if (curr_gen_number == max_generation)
34208 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
34210 // report bounds from heap_segment_mem (seg) to
34211 // generation_allocation_start (generation_of (max_generation-1))
34212 // for heap # heap_number
34214 fn(context, curr_gen_number, heap_segment_mem (seg),
34215 generation_allocation_start (hp->generation_of (max_generation-1)),
34216 generation_allocation_start (hp->generation_of (max_generation-1)) );
34219 else if (curr_gen_number != 0)
34221 //report bounds from generation_allocation_start (generation_of (curr_gen_number))
34222 // to generation_allocation_start (generation_of (curr_gen_number-1))
34223 // for heap # heap_number
34225 fn(context, curr_gen_number, generation_allocation_start (hp->generation_of (curr_gen_number)),
34226 generation_allocation_start (hp->generation_of (curr_gen_number-1)),
34227 generation_allocation_start (hp->generation_of (curr_gen_number-1)));
34231 //report bounds from generation_allocation_start (generation_of (curr_gen_number))
34232 // to heap_segment_allocated (ephemeral_heap_segment);
34233 // for heap # heap_number
34235 fn(context, curr_gen_number, generation_allocation_start (hp->generation_of (curr_gen_number)),
34236 heap_segment_allocated (hp->ephemeral_heap_segment),
34237 heap_segment_reserved (hp->ephemeral_heap_segment) );
34245 // Note that when logging is on it can take a long time to go through the free items.
34246 void gc_heap::print_free_list (int gen, heap_segment* seg)
34248 UNREFERENCED_PARAMETER(gen);
34249 UNREFERENCED_PARAMETER(seg);
34251 if (settings.concurrent == FALSE)
34253 uint8_t* seg_start = heap_segment_mem (seg);
34254 uint8_t* seg_end = heap_segment_allocated (seg);
34256 dprintf (3, ("Free list in seg %Ix:", seg_start));
34258 size_t total_free_item = 0;
34260 allocator* gen_allocator = generation_allocator (generation_of (gen));
34261 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
34263 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
34266 if (fo >= seg_start && fo < seg_end)
34270 size_t free_item_len = size(fo);
34272 dprintf (3, ("[%Ix, %Ix[:%Id",
34274 (size_t)(fo + free_item_len),
34278 fo = free_list_slot (fo);
34282 dprintf (3, ("total %Id free items", total_free_item));
34288 void gc_heap::descr_generations (BOOL begin_gc_p)
34291 UNREFERENCED_PARAMETER(begin_gc_p);
34295 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
34298 #ifdef MULTIPLE_HEAPS
34300 #endif //MULTIPLE_HEAPS
34302 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
34303 for (int n = max_generation; n >= 0; --n)
34305 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
34307 generation_allocation_start(generation_of(n)),
34308 generation_allocation_limit(generation_of(n)),
34309 generation_allocation_pointer(generation_of(n)));
34311 heap_segment* seg = generation_start_segment(generation_of(n));
34314 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
34315 heap_segment_mem(seg),
34316 heap_segment_allocated(seg),
34317 heap_segment_used(seg),
34318 heap_segment_committed(seg));
34319 seg = heap_segment_next(seg);
34323 #endif // STRESS_LOG
34326 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
34327 (size_t) lowest_address, (size_t) highest_address));
34328 #ifdef BACKGROUND_GC
34329 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
34330 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
34331 #endif //BACKGROUND_GC
34333 if (heap_number == 0)
34335 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
34338 for (int curr_gen_number = total_generation_count - 1; curr_gen_number >= 0; curr_gen_number--)
34340 size_t total_gen_size = generation_size (curr_gen_number);
34341 #ifdef SIMPLE_DPRINTF
34342 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
34343 (begin_gc_p ? "BEG" : "END"),
34344 settings.condemned_generation,
34347 dd_fragmentation (dynamic_data_of (curr_gen_number)),
34348 generation_free_list_space (generation_of (curr_gen_number)),
34349 generation_free_obj_space (generation_of (curr_gen_number)),
34351 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
34353 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
34354 (settings.heap_expansion ? "(EX)" : " "),
34355 (settings.promotion ? "Promotion" : "NoPromotion")));
34357 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
34359 size (generation_allocation_start (generation_of (curr_gen_number))),
34361 dd_fragmentation (dynamic_data_of (curr_gen_number))));
34362 #endif //SIMPLE_DPRINTF
34364 generation* gen = generation_of (curr_gen_number);
34365 heap_segment* seg = generation_start_segment (gen);
34366 while (seg && (seg != ephemeral_heap_segment))
34368 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
34370 (size_t)heap_segment_mem (seg),
34371 (size_t)heap_segment_allocated (seg),
34372 (size_t)heap_segment_committed (seg),
34373 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
34374 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
34375 print_free_list (curr_gen_number, seg);
34376 seg = heap_segment_next (seg);
34378 if (seg && (seg != generation_start_segment (gen)))
34380 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
34382 (size_t)heap_segment_mem (seg),
34383 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
34384 print_free_list (curr_gen_number, seg);
34389 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
34391 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
34392 (size_t)(((curr_gen_number == 0)) ?
34393 (heap_segment_allocated
34394 (generation_start_segment
34395 (generation_of (curr_gen_number)))) :
34396 (generation_allocation_start
34397 (generation_of (curr_gen_number - 1))))
34399 print_free_list (curr_gen_number, seg);
34406 //-----------------------------------------------------------------------------
34408 // VM Specific support
34410 //-----------------------------------------------------------------------------
34412 //Static member variables.
34413 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
34414 GCEvent *GCHeap::WaitForGCEvent = NULL;
34415 unsigned GCHeap::GcCondemnedGeneration = 0;
34416 size_t GCHeap::totalSurvivedSize = 0;
34417 #ifdef FEATURE_PREMORTEM_FINALIZATION
34418 CFinalize* GCHeap::m_Finalize = 0;
34419 BOOL GCHeap::GcCollectClasses = FALSE;
34420 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
34422 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34424 #ifndef MULTIPLE_HEAPS
34425 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
34426 int GCHeap::m_CurStressObj = 0;
34427 #endif // !MULTIPLE_HEAPS
34428 #endif // STRESS_HEAP
34429 #endif // FEATURE_REDHAWK
34431 #endif //FEATURE_PREMORTEM_FINALIZATION
34433 class NoGCRegionLockHolder
34436 NoGCRegionLockHolder()
34438 enter_spin_lock_noinstru(&g_no_gc_lock);
34441 ~NoGCRegionLockHolder()
34443 leave_spin_lock_noinstru(&g_no_gc_lock);
34447 // An explanation of locking for finalization:
34449 // Multiple threads allocate objects. During the allocation, they are serialized by
34450 // the AllocLock above. But they release that lock before they register the object
34451 // for finalization. That's because there is much contention for the alloc lock, but
34452 // finalization is presumed to be a rare case.
34454 // So registering an object for finalization must be protected by the FinalizeLock.
34456 // There is another logical queue that involves finalization. When objects registered
34457 // for finalization become unreachable, they are moved from the "registered" queue to
34458 // the "unreachable" queue. Note that this only happens inside a GC, so no other
34459 // threads can be manipulating either queue at that time. Once the GC is over and
34460 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
34461 // queue and call their finalizers. This dequeue operation is also protected with
34462 // the finalize lock.
34464 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
34465 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
34466 // when a GC is not in progress). The reason we share a lock with threads enqueuing
34467 // on the "registered" queue is that the "registered" and "unreachable" queues are
34470 // They are actually two regions of a longer list, which can only grow at one end.
34471 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
34472 // object at the boundary between the logical queues, out to the other end of the
34473 // unreachable queue -- where all growing takes place. Then you move the boundary
34474 // pointer so that the gap we created at the boundary is now on the "registered"
34475 // side rather than the "unreachable" side. Now the object can be placed into the
34476 // "registered" side at that point. This is much more efficient than doing moves
34477 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
34479 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
34480 // on the fact that the lock will only be taken for a brief period and that it will
34481 // never provoke or allow a GC while the lock is held. This is critical. If the
34482 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
34483 // allow a GC), then the Alloc client would have to GC protect a finalizable object
34484 // to protect against that eventuality. That is too slow!
34488 BOOL IsValidObject99(uint8_t *pObject)
34491 if (!((CObjectHeader*)pObject)->IsFree())
34492 ((CObjectHeader *) pObject)->Validate();
34493 #endif //VERIFY_HEAP
34497 #ifdef BACKGROUND_GC
34498 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
34500 uint8_t** range_beg,
34501 uint8_t** range_end)
34503 uint8_t* seg_start = heap_segment_mem (seg);
34504 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
34506 if ((seg_start < background_saved_highest_address) &&
34507 (seg_end > background_saved_lowest_address))
34509 *range_beg = max (seg_start, background_saved_lowest_address);
34510 *range_end = min (seg_end, background_saved_highest_address);
34519 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
34522 if (gc_heap::background_running_p() &&
34523 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
34525 uint8_t* range_beg = 0;
34526 uint8_t* range_end = 0;
34528 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
34530 size_t markw = mark_word_of (range_beg);
34531 size_t markw_end = mark_word_of (range_end);
34532 while (markw < markw_end)
34534 if (mark_array [markw])
34536 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
34537 markw, mark_array [markw], mark_word_address (markw)));
34542 uint8_t* p = mark_word_address (markw_end);
34543 while (p < range_end)
34545 assert (!(mark_array_marked (p)));
34550 #endif //VERIFY_HEAP
34553 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
34556 size_t start_mark_bit = mark_bit_of (obj) + 1;
34557 size_t end_mark_bit = mark_bit_of (obj + s);
34558 unsigned int startbit = mark_bit_bit (start_mark_bit);
34559 unsigned int endbit = mark_bit_bit (end_mark_bit);
34560 size_t startwrd = mark_bit_word (start_mark_bit);
34561 size_t endwrd = mark_bit_word (end_mark_bit);
34562 unsigned int result = 0;
34564 unsigned int firstwrd = ~(lowbits (~0, startbit));
34565 unsigned int lastwrd = ~(highbits (~0, endbit));
34567 if (startwrd == endwrd)
34569 unsigned int wrd = firstwrd & lastwrd;
34570 result = mark_array[startwrd] & wrd;
34578 // verify the first mark word is cleared.
34581 result = mark_array[startwrd] & firstwrd;
34589 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
34591 result = mark_array[wrdtmp];
34598 // set the last mark word.
34601 result = mark_array[endwrd] & lastwrd;
34607 #endif //VERIFY_HEAP
34610 void gc_heap::clear_all_mark_array()
34612 for (int i = max_generation; i < total_generation_count; i++)
34614 generation* gen = generation_of (i);
34615 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
34619 uint8_t* range_beg = 0;
34620 uint8_t* range_end = 0;
34622 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
34624 size_t markw = mark_word_of (range_beg);
34625 size_t markw_end = mark_word_of (range_end);
34626 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
34627 //num_dwords_written = markw_end - markw;
34629 size_t size_left = 0;
34631 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
34633 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
34635 size = (size_total & ~(sizeof(PTR_PTR) - 1));
34636 size_left = size_total - size;
34637 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
34644 memclr ((uint8_t*)&mark_array[markw], size);
34646 if (size_left != 0)
34648 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
34649 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
34651 *markw_to_clear = 0;
34657 seg = heap_segment_next_rw (seg);
34662 void gc_heap::verify_mark_array_cleared()
34665 if (gc_heap::background_running_p() &&
34666 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
34668 for (int i = max_generation; i < total_generation_count; i++)
34670 generation* gen = generation_of (i);
34671 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
34675 bgc_verify_mark_array_cleared (seg);
34676 seg = heap_segment_next_rw (seg);
34680 #endif //VERIFY_HEAP
34682 #endif //BACKGROUND_GC
34684 // This function is called to make sure we don't mess up the segment list
34685 // in SOH. It's called by:
34686 // 1) begin and end of ephemeral GCs
34687 // 2) during bgc sweep when we switch segments.
34688 void gc_heap::verify_soh_segment_list()
34691 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
34693 generation* gen = generation_of (max_generation);
34694 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
34695 heap_segment* last_seg = 0;
34699 seg = heap_segment_next_rw (seg);
34701 if (last_seg != ephemeral_heap_segment)
34706 #endif //VERIFY_HEAP
34709 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
34710 // it can be called at the end of the final marking; and at any point during background
34712 // NOTE - to be able to call this function during background sweep, we need to temporarily
34713 // NOT clear the mark array bits as we go.
34714 #ifdef BACKGROUND_GC
34715 void gc_heap::verify_partial()
34717 // Different ways to fail.
34718 BOOL mark_missed_p = FALSE;
34719 BOOL bad_ref_p = FALSE;
34720 BOOL free_ref_p = FALSE;
34722 for (int i = max_generation; i < total_generation_count; i++)
34724 generation* gen = generation_of (i);
34725 int align_const = get_alignment_constant (i == max_generation);
34726 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
34730 uint8_t* o = heap_segment_mem (seg);
34731 uint8_t* end = heap_segment_allocated (seg);
34735 size_t s = size (o);
34737 BOOL marked_p = background_object_marked (o, FALSE);
34741 go_through_object_cl (method_table (o), o, s, oo,
34745 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
34746 MethodTable *pMT = method_table (*oo);
34748 if (pMT == g_gc_pFreeObjectMethodTable)
34754 if (!pMT->SanityCheck())
34757 dprintf (3, ("Bad member of %Ix %Ix",
34758 (size_t)oo, (size_t)*oo));
34762 if (current_bgc_state == bgc_final_marking)
34764 if (marked_p && !background_object_marked (*oo, FALSE))
34766 mark_missed_p = TRUE;
34775 o = o + Align(s, align_const);
34777 seg = heap_segment_next_rw (seg);
34781 #endif //BACKGROUND_GC
34785 gc_heap::verify_free_lists ()
34787 for (int gen_num = 0; gen_num < total_generation_count; gen_num++)
34789 dprintf (3, ("Verifying free list for gen:%d", gen_num));
34790 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
34791 size_t sz = gen_alloc->first_bucket_size();
34792 bool verify_undo_slot = (gen_num != 0) && (gen_num <= max_generation) && !gen_alloc->discard_if_no_fit_p();
34794 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
34796 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
34800 if (!((CObjectHeader*)free_list)->IsFree())
34802 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
34803 (size_t)free_list));
34806 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
34807 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
34809 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
34810 (size_t)free_list));
34813 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
34815 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
34816 (size_t)free_list));
34819 if ((gen_num <= max_generation) && (object_gennum (free_list)!= gen_num))
34821 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
34822 (size_t)free_list));
34827 free_list = free_list_slot (free_list);
34829 //verify the sanity of the tail
34830 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
34831 if (!((tail == 0) || (tail == prev)))
34833 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
34838 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
34839 if ((head != 0) && (free_list_slot (head) != 0))
34841 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
34852 gc_heap::verify_heap (BOOL begin_gc_p)
34854 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
34856 #ifdef MULTIPLE_HEAPS
34857 t_join* current_join = &gc_t_join;
34858 #ifdef BACKGROUND_GC
34859 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
34861 // We always call verify_heap on entry of GC on the SVR GC threads.
34862 current_join = &bgc_t_join;
34864 #endif //BACKGROUND_GC
34865 #endif //MULTIPLE_HEAPS
34868 UNREFERENCED_PARAMETER(begin_gc_p);
34871 #ifdef BACKGROUND_GC
34872 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
34873 (begin_gc_p ? "BEG" : "END"),
34874 VolatileLoad(&settings.gc_index),
34875 (settings.concurrent ? "BGC" : (gc_heap::background_running_p() ? "FGC" : "NGC"))));
34877 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
34878 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
34879 #endif //BACKGROUND_GC
34881 #ifndef MULTIPLE_HEAPS
34882 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
34883 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
34887 #endif //MULTIPLE_HEAPS
34889 #ifdef BACKGROUND_GC
34890 //don't touch the memory because the program is allocating from it.
34891 if (!settings.concurrent)
34892 #endif //BACKGROUND_GC
34894 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
34896 // 0xaa the unused portions of segments.
34897 for (int i = max_generation; i < total_generation_count; i++)
34899 generation* gen1 = generation_of (i);
34900 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
34904 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
34905 if (heap_segment_used (seg1) > clear_start)
34907 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
34908 heap_segment_mem (seg1),
34910 heap_segment_used (seg1)));
34911 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
34912 (heap_segment_used (seg1) - clear_start));
34914 seg1 = heap_segment_next_rw (seg1);
34920 #ifdef MULTIPLE_HEAPS
34921 current_join->join(this, gc_join_verify_copy_table);
34922 if (current_join->joined())
34924 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
34925 for (int i = 0; i < n_heaps; i++)
34927 //copy the card and brick tables
34928 if (g_gc_card_table != g_heaps[i]->card_table)
34930 g_heaps[i]->copy_brick_card_table();
34934 current_join->restart();
34937 if (g_gc_card_table != card_table)
34938 copy_brick_card_table();
34939 #endif //MULTIPLE_HEAPS
34941 //verify that the generation structures makes sense
34943 generation* gen = generation_of (max_generation);
34945 assert (generation_allocation_start (gen) ==
34946 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
34947 int gen_num = max_generation-1;
34948 generation* prev_gen = gen;
34949 while (gen_num >= 0)
34951 gen = generation_of (gen_num);
34952 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
34953 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
34954 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
34956 if (generation_start_segment (prev_gen ) ==
34957 generation_start_segment (gen))
34959 assert (generation_allocation_start (prev_gen) <
34960 generation_allocation_start (gen));
34967 size_t total_objects_verified = 0;
34968 size_t total_objects_verified_deep = 0;
34970 BOOL bCurrentBrickInvalid = FALSE;
34971 size_t last_valid_brick = 0;
34972 size_t curr_brick = 0;
34973 size_t prev_brick = (size_t)-1;
34974 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
34975 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
34977 // go through all generations starting with the highest
34978 for (int curr_gen_num = total_generation_count - 1; curr_gen_num >= max_generation; curr_gen_num--)
34980 int align_const = get_alignment_constant (curr_gen_num == max_generation);
34981 BOOL large_brick_p = (curr_gen_num != max_generation);
34983 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num) ));
34987 uint8_t* curr_object = heap_segment_mem (seg);
34988 uint8_t* prev_object = 0;
34990 #ifdef BACKGROUND_GC
34991 BOOL consider_bgc_mark_p = FALSE;
34992 BOOL check_current_sweep_p = FALSE;
34993 BOOL check_saved_sweep_p = FALSE;
34994 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
34995 #endif //BACKGROUND_GC
34997 while (curr_object < heap_segment_allocated (seg))
34999 //if (is_mark_set (curr_object))
35001 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
35002 // FATAL_GC_ERROR();
35005 size_t s = size (curr_object);
35006 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
35009 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
35013 // handle generation boundaries within ephemeral segment
35014 if (seg == ephemeral_heap_segment)
35016 if ((curr_gen_num > 0) && (curr_object >= next_boundary))
35019 if (curr_gen_num > 0)
35021 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
35026 // If object is not in the youngest generation, then lets
35027 // verify that the brick table is correct....
35028 if (((seg != ephemeral_heap_segment) ||
35029 (brick_of(curr_object) < brick_of(begin_youngest))))
35031 curr_brick = brick_of(curr_object);
35033 // Brick Table Verification...
35035 // On brick transition
35036 // if brick is negative
35037 // verify that brick indirects to previous valid brick
35039 // set current brick invalid flag to be flipped if we
35040 // encounter an object at the correct place
35042 if (curr_brick != prev_brick)
35044 // If the last brick we were examining had positive
35045 // entry but we never found the matching object, then
35046 // we have a problem
35047 // If prev_brick was the last one of the segment
35048 // it's ok for it to be invalid because it is never looked at
35049 if (bCurrentBrickInvalid &&
35050 (curr_brick != brick_of (heap_segment_mem (seg))) &&
35051 !heap_segment_read_only_p (seg))
35053 dprintf (3, ("curr brick %Ix invalid", curr_brick));
35059 //large objects verify the table only if they are in
35061 if ((heap_segment_reserved (seg) <= highest_address) &&
35062 (heap_segment_mem (seg) >= lowest_address) &&
35063 brick_table [curr_brick] != 0)
35065 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
35066 curr_brick, (size_t)curr_object));
35071 bCurrentBrickInvalid = FALSE;
35076 // If the current brick contains a negative value make sure
35077 // that the indirection terminates at the last valid brick
35078 if (brick_table [curr_brick] <= 0)
35080 if (brick_table [curr_brick] == 0)
35082 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
35083 curr_brick, (size_t)curr_object));
35086 ptrdiff_t i = curr_brick;
35087 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
35088 (brick_table[i] < 0))
35090 i = i + brick_table[i];
35092 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
35094 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
35095 i, brick_of (heap_segment_mem (seg)),
35099 // if (i != last_valid_brick)
35100 // FATAL_GC_ERROR();
35101 bCurrentBrickInvalid = FALSE;
35103 else if (!heap_segment_read_only_p (seg))
35105 bCurrentBrickInvalid = TRUE;
35110 if (bCurrentBrickInvalid)
35112 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
35114 bCurrentBrickInvalid = FALSE;
35115 last_valid_brick = curr_brick;
35120 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
35122 #ifdef FEATURE_LOH_COMPACTION
35123 if ((curr_gen_num == loh_generation) && (prev_object != 0))
35125 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
35127 #endif //FEATURE_LOH_COMPACTION
35129 total_objects_verified++;
35131 BOOL can_verify_deep = TRUE;
35132 #ifdef BACKGROUND_GC
35133 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
35134 #endif //BACKGROUND_GC
35136 BOOL deep_verify_obj = can_verify_deep;
35137 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
35138 deep_verify_obj = FALSE;
35140 ((CObjectHeader*)curr_object)->ValidateHeap(deep_verify_obj);
35142 if (can_verify_deep)
35144 if (curr_gen_num > 0)
35146 BOOL need_card_p = FALSE;
35147 if (contain_pointers_or_collectible (curr_object))
35149 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
35150 size_t crd = card_of (curr_object);
35151 BOOL found_card_p = card_set_p (crd);
35153 #ifdef COLLECTIBLE_CLASS
35154 if (is_collectible(curr_object))
35156 uint8_t* class_obj = get_class_object (curr_object);
35157 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
35161 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
35162 card_of (curr_object), (size_t)curr_object, class_obj));
35168 #endif //COLLECTIBLE_CLASS
35170 if (contain_pointers(curr_object))
35172 go_through_object_nostart
35173 (method_table(curr_object), curr_object, s, oo,
35175 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
35177 crd = card_of ((uint8_t*)oo);
35178 found_card_p = card_set_p (crd);
35179 need_card_p = FALSE;
35181 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
35183 need_card_p = TRUE;
35186 if (need_card_p && !found_card_p)
35189 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
35190 card_of (curr_object), (size_t)curr_object,
35191 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
35197 if (need_card_p && !found_card_p)
35199 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
35200 card_of (curr_object), (size_t)curr_object,
35201 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
35206 total_objects_verified_deep++;
35210 prev_object = curr_object;
35211 prev_brick = curr_brick;
35212 curr_object = curr_object + Align(s, align_const);
35213 if (curr_object < prev_object)
35215 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
35220 if (curr_object > heap_segment_allocated(seg))
35222 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
35223 (size_t)curr_object, (size_t)seg));
35227 seg = heap_segment_next_in_range (seg);
35231 #ifdef BACKGROUND_GC
35232 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
35233 (settings.concurrent ? "BGC" : (gc_heap::background_running_p () ? "FGC" : "NGC")),
35234 (begin_gc_p ? "BEG" : "END"),
35235 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
35236 total_objects_verified, total_objects_verified_deep));
35237 if (current_c_gc_state != c_gc_state_planning)
35239 assert (total_objects_verified == total_objects_verified_deep);
35241 #endif //BACKGROUND_GC
35243 verify_free_lists();
35245 #ifdef FEATURE_PREMORTEM_FINALIZATION
35246 finalize_queue->CheckFinalizerObjects();
35247 #endif // FEATURE_PREMORTEM_FINALIZATION
35250 // to be consistent with handle table APIs pass a ScanContext*
35251 // to provide the heap number. the SC isn't complete though so
35252 // limit its scope to handle table verification.
35254 sc.thread_number = heap_number;
35255 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
35258 #ifdef MULTIPLE_HEAPS
35259 current_join->join(this, gc_join_verify_objects_done);
35260 if (current_join->joined())
35261 #endif //MULTIPLE_HEAPS
35263 GCToEEInterface::VerifySyncTableEntry();
35264 #ifdef MULTIPLE_HEAPS
35265 current_join->restart();
35266 #endif //MULTIPLE_HEAPS
35269 #ifdef BACKGROUND_GC
35270 if (settings.concurrent)
35272 verify_mark_array_cleared();
35274 dprintf (2,("GC%d(%s): Verifying heap - end",
35275 VolatileLoad(&settings.gc_index),
35276 (settings.concurrent ? "BGC" : (gc_heap::background_running_p() ? "FGC" : "NGC"))));
35278 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
35279 #endif //BACKGROUND_GC
35282 #endif //VERIFY_HEAP
35285 void GCHeap::ValidateObjectMember (Object* obj)
35288 size_t s = size (obj);
35289 uint8_t* o = (uint8_t*)obj;
35291 go_through_object_cl (method_table (obj), o, s, oo,
35293 uint8_t* child_o = *oo;
35296 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
35297 MethodTable *pMT = method_table (child_o);
35299 if (!pMT->SanityCheck()) {
35300 dprintf (3, ("Bad member of %Ix %Ix",
35301 (size_t)oo, (size_t)child_o));
35306 #endif // VERIFY_HEAP
35309 HRESULT GCHeap::StaticShutdown()
35313 GCScan::GcRuntimeStructuresValid (FALSE);
35315 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
35316 // threads except the one performing the shutdown.
35317 // ASSERT( !GcInProgress );
35319 // Guard against any more GC occurring and against any threads blocking
35320 // for GC to complete when the GC heap is gone. This fixes a race condition
35321 // where a thread in GC is destroyed as part of process destruction and
35322 // the remaining threads block for GC complete.
35325 //EnterAllocLock();
35327 //EnterFinalizeLock();
35330 // during shutdown lot of threads are suspended
35331 // on this even, we don't want to wake them up just yet
35332 //CloseHandle (WaitForGCEvent);
35334 //find out if the global card table hasn't been used yet
35335 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
35336 if (card_table_refcount (ct) == 0)
35338 destroy_card_table (ct);
35339 g_gc_card_table = nullptr;
35341 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
35342 g_gc_card_bundle_table = nullptr;
35344 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
35345 SoftwareWriteWatch::StaticClose();
35346 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
35349 //destroy all segments on the standby list
35350 while(gc_heap::segment_standby_list != 0)
35352 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
35353 #ifdef MULTIPLE_HEAPS
35354 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
35355 #else //MULTIPLE_HEAPS
35356 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
35357 #endif //MULTIPLE_HEAPS
35358 gc_heap::segment_standby_list = next_seg;
35362 #ifdef MULTIPLE_HEAPS
35364 for (int i = 0; i < gc_heap::n_heaps; i ++)
35366 delete gc_heap::g_heaps[i]->vm_heap;
35367 //destroy pure GC stuff
35368 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
35371 gc_heap::destroy_gc_heap (pGenGCHeap);
35373 #endif //MULTIPLE_HEAPS
35374 gc_heap::shutdown_gc();
35379 // Wait until a garbage collection is complete
35380 // returns NOERROR if wait was OK, other error code if failure.
35381 // WARNING: This will not undo the must complete state. If you are
35382 // in a must complete when you call this, you'd better know what you're
35385 #ifdef FEATURE_PREMORTEM_FINALIZATION
35387 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
35389 *pCFinalize = new (nothrow) CFinalize();
35390 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
35391 return E_OUTOFMEMORY;
35395 #endif // FEATURE_PREMORTEM_FINALIZATION
35397 // init the instance heap
35398 HRESULT GCHeap::Init(size_t hn)
35400 HRESULT hres = S_OK;
35402 #ifdef MULTIPLE_HEAPS
35403 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
35404 hres = E_OUTOFMEMORY;
35406 UNREFERENCED_PARAMETER(hn);
35407 if (!gc_heap::make_gc_heap())
35408 hres = E_OUTOFMEMORY;
35409 #endif //MULTIPLE_HEAPS
35415 //System wide initialization
35416 HRESULT GCHeap::Initialize()
35420 qpf = (uint64_t)GCToOSInterface::QueryPerformanceFrequency();
35421 qpf_ms = 1000.0 / (double)qpf;
35422 qpf_us = 1000.0 * 1000.0 / (double)qpf;
35424 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
35425 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
35426 assert(g_num_processors != 0);
35428 gc_heap::total_physical_mem = (size_t)GCConfig::GetGCTotalPhysicalMemory();
35429 if (gc_heap::total_physical_mem != 0)
35431 gc_heap::is_restricted_physical_mem = true;
35435 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&gc_heap::is_restricted_physical_mem);
35439 gc_heap::heap_hard_limit = (size_t)GCConfig::GetGCHeapHardLimit();
35440 gc_heap::heap_hard_limit_oh[0] = (size_t)GCConfig::GetGCHeapHardLimitSOH();
35441 gc_heap::heap_hard_limit_oh[1] = (size_t)GCConfig::GetGCHeapHardLimitLOH();
35442 gc_heap::heap_hard_limit_oh[2] = (size_t)GCConfig::GetGCHeapHardLimitPOH();
35444 if (gc_heap::heap_hard_limit_oh[0] || gc_heap::heap_hard_limit_oh[1] || gc_heap::heap_hard_limit_oh[2])
35446 if (!gc_heap::heap_hard_limit_oh[0])
35448 return E_INVALIDARG;
35450 if (!gc_heap::heap_hard_limit_oh[1])
35452 return E_INVALIDARG;
35454 if (gc_heap::heap_hard_limit_oh[2] < min_segment_size_hard_limit)
35456 gc_heap::heap_hard_limit_oh[2] = min_segment_size_hard_limit;
35458 gc_heap::heap_hard_limit = gc_heap::heap_hard_limit_oh[0] + gc_heap::heap_hard_limit_oh[1] + gc_heap::heap_hard_limit_oh[2];
35462 uint32_t percent_of_mem_soh = (uint32_t)GCConfig::GetGCHeapHardLimitSOHPercent();
35463 uint32_t percent_of_mem_loh = (uint32_t)GCConfig::GetGCHeapHardLimitLOHPercent();
35464 uint32_t percent_of_mem_poh = (uint32_t)GCConfig::GetGCHeapHardLimitPOHPercent();
35465 if (percent_of_mem_soh || percent_of_mem_loh || percent_of_mem_poh)
35467 if ((percent_of_mem_soh <= 0) || (percent_of_mem_soh >= 100))
35469 return E_INVALIDARG;
35471 if ((percent_of_mem_loh <= 0) || (percent_of_mem_loh >= 100))
35473 return E_INVALIDARG;
35475 else if ((percent_of_mem_poh < 0) || (percent_of_mem_poh >= 100))
35477 return E_INVALIDARG;
35479 if ((percent_of_mem_soh + percent_of_mem_loh + percent_of_mem_poh) >= 100)
35481 return E_INVALIDARG;
35483 gc_heap::heap_hard_limit_oh[0] = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem_soh / (uint64_t)100);
35484 gc_heap::heap_hard_limit_oh[1] = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem_loh / (uint64_t)100);
35485 if (percent_of_mem_poh == 0)
35487 gc_heap::heap_hard_limit_oh[2] = min_segment_size_hard_limit;
35491 gc_heap::heap_hard_limit_oh[2] = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem_poh / (uint64_t)100);
35493 gc_heap::heap_hard_limit = gc_heap::heap_hard_limit_oh[0] + gc_heap::heap_hard_limit_oh[1] + gc_heap::heap_hard_limit_oh[2];
35497 if (!(gc_heap::heap_hard_limit))
35499 uint32_t percent_of_mem = (uint32_t)GCConfig::GetGCHeapHardLimitPercent();
35500 if ((percent_of_mem > 0) && (percent_of_mem < 100))
35502 gc_heap::heap_hard_limit = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem / (uint64_t)100);
35506 // If the hard limit is specified, the user is saying even if the process is already
35507 // running in a container, use this limit for the GC heap.
35508 if (!(gc_heap::heap_hard_limit))
35510 if (gc_heap::is_restricted_physical_mem)
35512 uint64_t physical_mem_for_gc = gc_heap::total_physical_mem * (uint64_t)75 / (uint64_t)100;
35513 gc_heap::heap_hard_limit = (size_t)max ((20 * 1024 * 1024), physical_mem_for_gc);
35516 #endif //HOST_64BIT
35519 uint32_t nhp_from_config = 0;
35521 #ifdef MULTIPLE_HEAPS
35522 AffinitySet config_affinity_set;
35523 GCConfigStringHolder cpu_index_ranges_holder(GCConfig::GetGCHeapAffinitizeRanges());
35525 if (!ParseGCHeapAffinitizeRanges(cpu_index_ranges_holder.Get(), &config_affinity_set))
35527 return CLR_E_GC_BAD_AFFINITY_CONFIG_FORMAT;
35530 uintptr_t config_affinity_mask = static_cast<uintptr_t>(GCConfig::GetGCHeapAffinitizeMask());
35531 const AffinitySet* process_affinity_set = GCToOSInterface::SetGCThreadsAffinitySet(config_affinity_mask, &config_affinity_set);
35533 if (process_affinity_set->IsEmpty())
35535 return CLR_E_GC_BAD_AFFINITY_CONFIG;
35538 if ((cpu_index_ranges_holder.Get() != nullptr)
35539 #ifdef TARGET_WINDOWS
35540 || (config_affinity_mask != 0)
35544 affinity_config_specified_p = true;
35547 nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
35549 g_num_active_processors = GCToOSInterface::GetCurrentProcessCpuCount();
35551 if (nhp_from_config)
35553 // Even when the user specifies a heap count, it should not be more
35554 // than the number of procs this process can use.
35555 nhp_from_config = min (nhp_from_config, g_num_active_processors);
35558 nhp = ((nhp_from_config == 0) ? g_num_active_processors : nhp_from_config);
35560 nhp = min (nhp, MAX_SUPPORTED_CPUS);
35561 #ifndef FEATURE_REDHAWK
35562 gc_heap::gc_thread_no_affinitize_p = (gc_heap::heap_hard_limit ? !affinity_config_specified_p : (GCConfig::GetNoAffinitize() != 0));
35564 if (!(gc_heap::gc_thread_no_affinitize_p))
35566 uint32_t num_affinitized_processors = (uint32_t)process_affinity_set->Count();
35568 if (num_affinitized_processors != 0)
35570 nhp = min(nhp, num_affinitized_processors);
35572 #ifndef TARGET_WINDOWS
35573 // Limit the GC heaps to the number of processors available in the system.
35574 nhp = min (nhp, GCToOSInterface::GetTotalProcessorCount());
35575 #endif // !TARGET_WINDOWS
35577 #endif //!FEATURE_REDHAWK
35578 #endif //MULTIPLE_HEAPS
35580 size_t seg_size = 0;
35581 size_t large_seg_size = 0;
35582 size_t pin_seg_size = 0;
35584 if (gc_heap::heap_hard_limit)
35586 gc_heap::use_large_pages_p = GCConfig::GetGCLargePages();
35587 if (gc_heap::heap_hard_limit_oh[0])
35589 #ifdef MULTIPLE_HEAPS
35590 if (nhp_from_config == 0)
35592 for (int i = 0; i < (total_oh_count - 1); i++)
35594 uint32_t nhp_oh = (uint32_t)(gc_heap::heap_hard_limit_oh[i] / min_segment_size_hard_limit);
35595 nhp = min (nhp, nhp_oh);
35603 seg_size = gc_heap::heap_hard_limit_oh[0] / nhp;
35604 large_seg_size = gc_heap::heap_hard_limit_oh[1] / nhp;
35605 pin_seg_size = gc_heap::heap_hard_limit_oh[2] / nhp;
35607 size_t aligned_seg_size = align_on_segment_hard_limit (seg_size);
35608 size_t aligned_large_seg_size = align_on_segment_hard_limit (large_seg_size);
35609 size_t aligned_pin_seg_size = align_on_segment_hard_limit (pin_seg_size);
35611 if (!gc_heap::use_large_pages_p)
35613 aligned_seg_size = round_up_power2 (aligned_seg_size);
35614 aligned_large_seg_size = round_up_power2 (aligned_large_seg_size);
35615 aligned_pin_seg_size = round_up_power2 (aligned_pin_seg_size);
35618 size_t seg_size_from_config = (size_t)GCConfig::GetSegmentSize();
35619 if (seg_size_from_config)
35621 size_t aligned_seg_size_config = (gc_heap::use_large_pages_p ? align_on_segment_hard_limit (seg_size) : round_up_power2 (seg_size_from_config));
35622 aligned_seg_size = max (aligned_seg_size, aligned_seg_size_config);
35623 aligned_large_seg_size = max (aligned_large_seg_size, aligned_seg_size_config);
35624 aligned_pin_seg_size = max (aligned_pin_seg_size, aligned_seg_size_config);
35627 seg_size = aligned_seg_size;
35628 gc_heap::soh_segment_size = seg_size;
35629 large_seg_size = aligned_large_seg_size;
35630 pin_seg_size = aligned_pin_seg_size;
35634 seg_size = gc_heap::get_segment_size_hard_limit (&nhp, (nhp_from_config == 0));
35635 gc_heap::soh_segment_size = seg_size;
35636 large_seg_size = gc_heap::use_large_pages_p ? seg_size : seg_size * 2;
35637 pin_seg_size = large_seg_size;
35639 if (gc_heap::use_large_pages_p)
35640 gc_heap::min_segment_size = min_segment_size_hard_limit;
35644 seg_size = get_valid_segment_size();
35645 gc_heap::soh_segment_size = seg_size;
35646 large_seg_size = get_valid_segment_size (TRUE);
35647 pin_seg_size = large_seg_size;
35649 assert (g_theGCHeap->IsValidSegmentSize (seg_size));
35650 assert (g_theGCHeap->IsValidSegmentSize (large_seg_size));
35651 assert (g_theGCHeap->IsValidSegmentSize (pin_seg_size));
35653 dprintf (1, ("%d heaps, soh seg size: %Id mb, loh: %Id mb\n",
35655 (seg_size / (size_t)1024 / 1024),
35656 (large_seg_size / 1024 / 1024)));
35658 gc_heap::min_uoh_segment_size = min (large_seg_size, pin_seg_size);
35660 if (gc_heap::min_segment_size == 0)
35662 gc_heap::min_segment_size = min (seg_size, gc_heap::min_uoh_segment_size);
35664 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
35666 #ifdef MULTIPLE_HEAPS
35667 gc_heap::n_heaps = nhp;
35668 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*loh_segment_size*/, pin_seg_size /*poh_segment_size*/, nhp);
35670 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*loh_segment_size*/, pin_seg_size /*poh_segment_size*/);
35671 #endif //MULTIPLE_HEAPS
35676 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
35677 #ifndef MULTIPLE_HEAPS
35678 gc_heap::mem_one_percent /= g_num_processors;
35679 #endif //!MULTIPLE_HEAPS
35681 uint32_t highmem_th_from_config = (uint32_t)GCConfig::GetGCHighMemPercent();
35682 if (highmem_th_from_config)
35684 gc_heap::high_memory_load_th = min (99, highmem_th_from_config);
35685 gc_heap::v_high_memory_load_th = min (99, (highmem_th_from_config + 7));
35689 // We should only use this if we are in the "many process" mode which really is only applicable
35690 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
35691 // For now I am using an estimate to calculate these numbers but this should really be obtained
35692 // programmatically going forward.
35693 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
35694 // I am assuming 3 in part due to the "very high memory load" is 97%.
35695 int available_mem_th = 10;
35696 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
35698 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(GCToOSInterface::GetTotalProcessorCount()));
35699 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
35702 gc_heap::high_memory_load_th = 100 - available_mem_th;
35703 gc_heap::v_high_memory_load_th = 97;
35706 gc_heap::m_high_memory_load_th = min ((gc_heap::high_memory_load_th + 5), gc_heap::v_high_memory_load_th);
35708 gc_heap::pm_stress_on = (GCConfig::GetGCProvModeStress() != 0);
35710 #if defined(HOST_64BIT)
35711 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
35712 #endif // HOST_64BIT
35714 WaitForGCEvent = new (nothrow) GCEvent;
35716 if (!WaitForGCEvent)
35718 return E_OUTOFMEMORY;
35721 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
35726 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
35727 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
35728 if (GCStress<cfg_any>::IsEnabled()) {
35729 for (int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
35731 m_StressObjs[i] = CreateGlobalHandle(0);
35733 m_CurStressObj = 0;
35735 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
35736 #endif // FEATURE_REDHAWK
35738 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
35740 #ifdef MULTIPLE_HEAPS
35742 for (uint32_t i = 0; i < nhp; i++)
35744 GCHeap* Hp = new (nothrow) GCHeap();
35746 return E_OUTOFMEMORY;
35748 if ((hr = Hp->Init (i))!= S_OK)
35754 heap_select::init_numa_node_to_heap_map (nhp);
35756 // If we have more active processors than heaps we still want to initialize some of the
35757 // mapping for the rest of the active processors because user threads can still run on
35758 // them which means it's important to know their numa nodes and map them to a reasonable
35759 // heap, ie, we wouldn't want to have all such procs go to heap 0.
35760 if (g_num_active_processors > nhp)
35761 heap_select::distribute_other_procs();
35763 gc_heap* hp = gc_heap::g_heaps[0];
35765 dynamic_data* gen0_dd = hp->dynamic_data_of (0);
35766 gc_heap::min_gen0_balance_delta = (dd_min_size (gen0_dd) >> 3);
35768 #ifdef HEAP_BALANCE_INSTRUMENTATION
35769 cpu_group_enabled_p = GCToOSInterface::CanEnableGCCPUGroups();
35771 if (!GCToOSInterface::GetNumaInfo (&total_numa_nodes_on_machine, &procs_per_numa_node))
35773 total_numa_nodes_on_machine = 1;
35775 // Note that if we are in cpu groups we need to take the way proc index is calculated
35776 // into consideration. It would mean we have more than 64 procs on one numa node -
35777 // this is mostly for testing (if we want to simulate no numa on a numa system).
35778 // see vm\gcenv.os.cpp GroupProcNo implementation.
35779 if (GCToOSInterface::GetCPUGroupInfo (&total_cpu_groups_on_machine, &procs_per_cpu_group))
35780 procs_per_numa_node = procs_per_cpu_group + ((total_cpu_groups_on_machine - 1) << 6);
35782 procs_per_numa_node = g_num_processors;
35784 hb_info_numa_nodes = new (nothrow) heap_balance_info_numa[total_numa_nodes_on_machine];
35785 dprintf (HEAP_BALANCE_LOG, ("total: %d, numa: %d", g_num_processors, total_numa_nodes_on_machine));
35787 int hb_info_size_per_proc = sizeof (heap_balance_info_proc);
35789 for (int numa_node_index = 0; numa_node_index < total_numa_nodes_on_machine; numa_node_index++)
35791 int hb_info_size_per_node = hb_info_size_per_proc * procs_per_numa_node;
35792 uint8_t* numa_mem = (uint8_t*)GCToOSInterface::VirtualReserve (hb_info_size_per_node, 0, 0, numa_node_index);
35795 if (!GCToOSInterface::VirtualCommit (numa_mem, hb_info_size_per_node, numa_node_index))
35798 heap_balance_info_proc* hb_info_procs = (heap_balance_info_proc*)numa_mem;
35799 hb_info_numa_nodes[numa_node_index].hb_info_procs = hb_info_procs;
35801 for (int proc_index = 0; proc_index < (int)procs_per_numa_node; proc_index++)
35803 heap_balance_info_proc* hb_info_proc = &hb_info_procs[proc_index];
35804 hb_info_proc->count = default_max_hb_heap_balance_info;
35805 hb_info_proc->index = 0;
35808 #endif //HEAP_BALANCE_INSTRUMENTATION
35811 #endif //MULTIPLE_HEAPS
35815 GCScan::GcRuntimeStructuresValid (TRUE);
35817 GCToEEInterface::DiagUpdateGenerationBounds();
35824 // GC callback functions
35825 bool GCHeap::IsPromoted(Object* object)
35830 ((CObjectHeader*)object)->Validate();
35834 uint8_t* o = (uint8_t*)object;
35836 if (gc_heap::settings.condemned_generation == max_generation)
35838 #ifdef MULTIPLE_HEAPS
35839 gc_heap* hp = gc_heap::g_heaps[0];
35841 gc_heap* hp = pGenGCHeap;
35842 #endif //MULTIPLE_HEAPS
35844 #ifdef BACKGROUND_GC
35845 if (gc_heap::settings.concurrent)
35847 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
35848 hp->background_marked (o));
35852 #endif //BACKGROUND_GC
35854 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
35855 || hp->is_mark_set (o));
35860 gc_heap* hp = gc_heap::heap_of (o);
35861 return (!((o < hp->gc_high) && (o >= hp->gc_low))
35862 || hp->is_mark_set (o));
35866 size_t GCHeap::GetPromotedBytes(int heap_index)
35868 #ifdef BACKGROUND_GC
35869 if (gc_heap::settings.concurrent)
35871 return gc_heap::bpromoted_bytes (heap_index);
35874 #endif //BACKGROUND_GC
35876 return gc_heap::promoted_bytes (heap_index);
35880 void GCHeap::SetYieldProcessorScalingFactor (float scalingFactor)
35882 assert (yp_spin_count_unit != 0);
35883 int saved_yp_spin_count_unit = yp_spin_count_unit;
35884 yp_spin_count_unit = (int)((float)yp_spin_count_unit * scalingFactor / (float)9);
35886 // It's very suspicious if it becomes 0
35887 if (yp_spin_count_unit == 0)
35889 yp_spin_count_unit = saved_yp_spin_count_unit;
35893 unsigned int GCHeap::WhichGeneration (Object* object)
35895 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
35896 unsigned int g = hp->object_gennum ((uint8_t*)object);
35897 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
35901 bool GCHeap::IsEphemeral (Object* object)
35903 uint8_t* o = (uint8_t*)object;
35904 gc_heap* hp = gc_heap::heap_of (o);
35905 return !!hp->ephemeral_pointer_p (o);
35908 // Return NULL if can't find next object. When EE is not suspended,
35909 // the result is not accurate: if the input arg is in gen0, the function could
35910 // return zeroed out memory as next object
35911 Object * GCHeap::NextObj (Object * object)
35914 uint8_t* o = (uint8_t*)object;
35916 #ifndef FEATURE_BASICFREEZE
35917 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
35921 #endif //!FEATURE_BASICFREEZE
35923 heap_segment * hs = gc_heap::find_segment (o, FALSE);
35929 BOOL large_object_p = heap_segment_uoh_p (hs);
35930 if (large_object_p)
35931 return NULL; //could be racing with another core allocating.
35932 #ifdef MULTIPLE_HEAPS
35933 gc_heap* hp = heap_segment_heap (hs);
35934 #else //MULTIPLE_HEAPS
35936 #endif //MULTIPLE_HEAPS
35937 unsigned int g = hp->object_gennum ((uint8_t*)object);
35938 if ((g == 0) && hp->settings.demotion)
35939 return NULL;//could be racing with another core allocating.
35940 int align_const = get_alignment_constant (!large_object_p);
35941 uint8_t* nextobj = o + Align (size (o), align_const);
35942 if (nextobj <= o) // either overflow or 0 sized object.
35947 if ((nextobj < heap_segment_mem(hs)) ||
35948 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
35949 (nextobj >= hp->alloc_allocated))
35954 return (Object *)nextobj;
35957 #endif // VERIFY_HEAP
35960 // returns TRUE if the pointer is in one of the GC heaps.
35961 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
35963 uint8_t* object = (uint8_t*) vpObject;
35964 #ifndef FEATURE_BASICFREEZE
35965 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
35967 #endif //!FEATURE_BASICFREEZE
35969 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
35973 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
35975 THREAD_NUMBER_FROM_CONTEXT;
35976 #ifndef MULTIPLE_HEAPS
35977 const int thread = 0;
35978 #endif //!MULTIPLE_HEAPS
35980 uint8_t* o = (uint8_t*)*ppObject;
35985 #ifdef DEBUG_DestroyedHandleValue
35986 // we can race with destroy handle during concurrent scan
35987 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
35989 #endif //DEBUG_DestroyedHandleValue
35993 gc_heap* hp = gc_heap::heap_of (o);
35995 if ((o < hp->gc_low) || (o >= hp->gc_high))
36000 dprintf (3, ("Promote %Ix", (size_t)o));
36002 if (flags & GC_CALL_INTERIOR)
36004 if ((o = hp->find_object (o)) == 0)
36010 #ifdef FEATURE_CONSERVATIVE_GC
36011 // For conservative GC, a value on stack may point to middle of a free object.
36012 // In this case, we don't need to promote the pointer.
36013 if (GCConfig::GetConservativeGC()
36014 && ((CObjectHeader*)o)->IsFree())
36021 ((CObjectHeader*)o)->Validate();
36023 UNREFERENCED_PARAMETER(sc);
36026 if (flags & GC_CALL_PINNED)
36027 hp->pin_object (o, (uint8_t**) ppObject);
36029 #ifdef STRESS_PINNING
36030 if ((++n_promote % 20) == 1)
36031 hp->pin_object (o, (uint8_t**) ppObject);
36032 #endif //STRESS_PINNING
36034 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
36036 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
36039 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
36042 UNREFERENCED_PARAMETER(sc);
36044 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
36046 THREAD_NUMBER_FROM_CONTEXT;
36048 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
36049 dprintf (3, ("R: %Ix", (size_t)ppObject));
36054 gc_heap* hp = gc_heap::heap_of (object);
36057 if (!(flags & GC_CALL_INTERIOR))
36059 // We cannot validate this object if it's in the condemned gen because it could
36060 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
36061 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
36063 ((CObjectHeader*)object)->Validate(FALSE);
36068 dprintf (3, ("Relocate %Ix\n", (size_t)object));
36072 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
36074 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
36079 if (gc_heap::loh_object_p (object))
36081 pheader = hp->find_object (object);
36087 ptrdiff_t ref_offset = object - pheader;
36088 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
36089 *ppObject = (Object*)(pheader + ref_offset);
36096 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
36097 *ppObject = (Object*)pheader;
36100 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
36103 /*static*/ bool GCHeap::IsLargeObject(Object *pObj)
36105 return size( pObj ) >= loh_size_threshold;
36108 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
36111 void StressHeapDummy ();
36113 // CLRRandom implementation can produce FPU exceptions if
36114 // the test/application run by CLR is enabling any FPU exceptions.
36115 // We want to avoid any unexpected exception coming from stress
36116 // infrastructure, so CLRRandom is not an option.
36117 // The code below is a replicate of CRT rand() implementation.
36118 // Using CRT rand() is not an option because we will interfere with the user application
36119 // that may also use it.
36120 int StressRNG(int iMaxValue)
36122 static BOOL bisRandInit = FALSE;
36123 static int lHoldrand = 1L;
36127 lHoldrand = (int)time(NULL);
36128 bisRandInit = TRUE;
36130 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
36131 return randValue % iMaxValue;
36133 #endif // STRESS_HEAP
36134 #endif // !FEATURE_REDHAWK
36136 // free up object so that things will move and then do a GC
36137 //return TRUE if GC actually happens, otherwise FALSE
36138 bool GCHeap::StressHeap(gc_alloc_context * context)
36140 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
36141 alloc_context* acontext = static_cast<alloc_context*>(context);
36142 assert(context != nullptr);
36144 // if GC stress was dynamically disabled during this run we return FALSE
36145 if (!GCStressPolicy::IsEnabled())
36149 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
36155 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
36157 || g_pConfig->FastGCStressLevel() > 1
36160 if (!Thread::UniqueStack(&acontext)) {
36165 #ifdef BACKGROUND_GC
36166 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
36167 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
36171 #endif //BACKGROUND_GC
36173 if (g_pStringClass == 0)
36175 // If the String class has not been loaded, dont do any stressing. This should
36176 // be kept to a minimum to get as complete coverage as possible.
36177 _ASSERTE(g_fEEInit);
36181 #ifndef MULTIPLE_HEAPS
36182 static int32_t OneAtATime = -1;
36184 // Only bother with this if the stress level is big enough and if nobody else is
36185 // doing it right now. Note that some callers are inside the AllocLock and are
36186 // guaranteed synchronized. But others are using AllocationContexts and have no
36187 // particular synchronization.
36189 // For this latter case, we want a very high-speed way of limiting this to one
36190 // at a time. A secondary advantage is that we release part of our StressObjs
36191 // buffer sparingly but just as effectively.
36193 if (Interlocked::Increment(&OneAtATime) == 0 &&
36194 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
36198 // If the current string is used up
36199 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
36201 // Populate handles with strings
36202 int i = m_CurStressObj;
36203 while(HndFetchHandle(m_StressObjs[i]) == 0)
36205 _ASSERTE(m_StressObjs[i] != 0);
36206 unsigned strLen = ((unsigned)loh_size_threshold - 32) / sizeof(WCHAR);
36207 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
36209 // update the cached type handle before allocating
36210 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
36211 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext, /*flags*/ 0);
36214 str->SetMethodTable (g_pStringClass);
36215 str->SetStringLength (strLen);
36216 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
36218 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
36219 if (i == m_CurStressObj) break;
36222 // advance the current handle to the next string
36223 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
36226 // Get the current string
36227 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
36230 // Chop off the end of the string and form a new object out of it.
36231 // This will 'free' an object at the beginning of the heap, which will
36232 // force data movement. Note that we can only do this so many times.
36233 // before we have to move on to the next string.
36234 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
36235 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
36237 unsigned sizeToNextObj = (unsigned)Align(size(str));
36238 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
36239 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
36240 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
36244 // Let the string itself become garbage.
36245 // will be realloced next time around
36246 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
36250 Interlocked::Decrement(&OneAtATime);
36251 #endif // !MULTIPLE_HEAPS
36252 if (IsConcurrentGCEnabled())
36254 int rgen = StressRNG(10);
36256 // gen0:gen1:gen2 distribution: 40:40:20
36259 else if (rgen >= 4)
36264 GarbageCollectTry (rgen, FALSE, collection_gcstress);
36268 GarbageCollect(max_generation, FALSE, collection_gcstress);
36273 UNREFERENCED_PARAMETER(context);
36275 #endif //STRESS_HEAP && !FEATURE_REDHAWK
36278 #ifdef FEATURE_PREMORTEM_FINALIZATION
36279 #define REGISTER_FOR_FINALIZATION(_object, _size) \
36280 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
36281 #else // FEATURE_PREMORTEM_FINALIZATION
36282 #define REGISTER_FOR_FINALIZATION(_object, _size) true
36283 #endif // FEATURE_PREMORTEM_FINALIZATION
36285 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
36286 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
36288 STRESS_LOG_OOM_STACK(_size); \
36293 #ifdef FEATURE_64BIT_ALIGNMENT
36295 // Allocate small object with an alignment requirement of 8-bytes.
36296 Object* AllocAlign8(alloc_context* acontext, gc_heap* hp, size_t size, uint32_t flags)
36303 Object* newAlloc = NULL;
36305 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
36306 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
36307 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
36308 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
36310 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
36311 // lock at this point).
36312 uint8_t* result = acontext->alloc_ptr;
36314 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
36316 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
36318 // Yes, we can just go ahead and make the allocation.
36319 newAlloc = (Object*) hp->allocate (size, acontext, flags);
36320 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
36324 // No, either the next available address is not aligned in the way we require it or there's
36325 // not enough space to allocate an object of the required size. In both cases we allocate a
36326 // padding object (marked as a free object). This object's size is such that it will reverse
36327 // the alignment of the next header (asserted below).
36329 // We allocate both together then decide based on the result whether we'll format the space as
36330 // free object + real object or real object + free object.
36331 ASSERT((Align(min_obj_size) & 7) == 4);
36332 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext, flags);
36335 if (((size_t)freeobj & 7) == desiredAlignment)
36337 // New allocation has desired alignment, return this one and place the free object at the
36338 // end of the allocated space.
36339 newAlloc = (Object*)freeobj;
36340 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
36344 // New allocation is still mis-aligned, format the initial space as a free object and the
36345 // rest of the space should be correctly aligned for the real object.
36346 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
36347 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
36348 if (flags & GC_ALLOC_ZEROING_OPTIONAL)
36350 // clean the syncblock of the aligned object.
36351 *(((PTR_PTR)newAlloc)-1) = 0;
36354 freeobj->SetFree(min_obj_size);
36360 #endif // FEATURE_64BIT_ALIGNMENT
36363 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
36372 Object* newAlloc = NULL;
36373 alloc_context* acontext = static_cast<alloc_context*>(context);
36375 #ifdef MULTIPLE_HEAPS
36376 if (acontext->get_alloc_heap() == 0)
36378 AssignHeap (acontext);
36379 assert (acontext->get_alloc_heap());
36381 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
36383 gc_heap* hp = pGenGCHeap;
36385 // prefix complains about us dereferencing hp in wks build even though we only access static members
36386 // this way. not sure how to shut it up except for this ugly workaround:
36387 PREFIX_ASSUME(hp != NULL);
36389 #endif //MULTIPLE_HEAPS
36391 assert(size < loh_size_threshold || (flags & GC_ALLOC_LARGE_OBJECT_HEAP));
36393 if (flags & GC_ALLOC_USER_OLD_HEAP)
36395 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
36396 // support mis-aligned object headers so we can't support biased headers. Luckily for us
36397 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
36398 // these can never get large enough to be allocated on the LOH.
36399 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
36400 ASSERT(65536 < loh_size_threshold);
36402 int gen_num = (flags & GC_ALLOC_PINNED_OBJECT_HEAP) ? poh_generation : loh_generation;
36403 newAlloc = (Object*) hp->allocate_uoh_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), flags, gen_num, acontext->alloc_bytes_uoh);
36404 ASSERT(((size_t)newAlloc & 7) == 0);
36406 #ifdef FEATURE_STRUCTALIGN
36407 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
36408 #endif // FEATURE_STRUCTALIGN
36412 #ifdef FEATURE_64BIT_ALIGNMENT
36413 if (flags & GC_ALLOC_ALIGN8)
36415 newAlloc = AllocAlign8 (acontext, hp, size, flags);
36419 assert ((flags & GC_ALLOC_ALIGN8) == 0);
36422 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext, flags);
36425 #ifdef FEATURE_STRUCTALIGN
36426 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
36427 #endif // FEATURE_STRUCTALIGN
36430 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
36435 GCHeap::FixAllocContext (gc_alloc_context* context, void* arg, void *heap)
36437 alloc_context* acontext = static_cast<alloc_context*>(context);
36438 #ifdef MULTIPLE_HEAPS
36441 acontext->alloc_count = 0;
36443 uint8_t * alloc_ptr = acontext->alloc_ptr;
36448 // The acontext->alloc_heap can be out of sync with the ptrs because
36449 // of heap re-assignment in allocate
36450 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
36452 gc_heap* hp = pGenGCHeap;
36453 #endif //MULTIPLE_HEAPS
36455 if (heap == NULL || heap == hp)
36457 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
36458 get_alignment_constant(TRUE));
36463 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
36465 uint8_t *o = (uint8_t*)pInteriorPtr;
36467 gc_heap* hp = gc_heap::heap_of (o);
36469 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
36470 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
36472 if (o >= lowest && o < highest)
36474 o = hp->find_object (o);
36481 return (Object *)o;
36484 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
36486 if (dd_new_allocation (dd) < 0)
36491 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
36499 //----------------------------------------------------------------------------
36500 // #GarbageCollector
36502 // API to ensure that a complete new garbage collection takes place
36505 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
36507 #if defined(HOST_64BIT)
36510 size_t total_allocated = 0;
36511 size_t total_desired = 0;
36512 #ifdef MULTIPLE_HEAPS
36514 for (hn = 0; hn < gc_heap::n_heaps; hn++)
36516 gc_heap* hp = gc_heap::g_heaps [hn];
36517 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
36518 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
36519 dd_new_allocation (hp->dynamic_data_of (0));
36522 gc_heap* hp = pGenGCHeap;
36523 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
36524 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
36525 dd_new_allocation (hp->dynamic_data_of (0));
36526 #endif //MULTIPLE_HEAPS
36528 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
36530 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
36531 total_allocated, total_desired));
36536 #endif // HOST_64BIT
36538 #ifdef MULTIPLE_HEAPS
36539 gc_heap* hpt = gc_heap::g_heaps[0];
36542 #endif //MULTIPLE_HEAPS
36544 generation = (generation < 0) ? max_generation : min (generation, max_generation);
36545 dynamic_data* dd = hpt->dynamic_data_of (generation);
36547 #ifdef BACKGROUND_GC
36548 if (gc_heap::background_running_p())
36550 if ((mode == collection_optimized) || (mode & collection_non_blocking))
36554 if (mode & collection_blocking)
36556 pGenGCHeap->background_gc_wait();
36557 if (mode & collection_optimized)
36563 #endif //BACKGROUND_GC
36565 if (mode & collection_optimized)
36567 if (pGenGCHeap->gc_started)
36573 BOOL should_collect = FALSE;
36574 BOOL should_check_uoh = (generation == max_generation);
36575 #ifdef MULTIPLE_HEAPS
36576 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
36578 dynamic_data* dd1 = gc_heap::g_heaps [heap_number]->dynamic_data_of (generation);
36579 should_collect = should_collect_optimized (dd1, low_memory_p);
36580 if (should_check_uoh)
36582 for (int i = uoh_start_generation; i < total_generation_count && !should_collect; i++)
36584 should_collect = should_collect_optimized (gc_heap::g_heaps [heap_number]->dynamic_data_of (i), low_memory_p);
36588 if (should_collect)
36592 should_collect = should_collect_optimized (dd, low_memory_p);
36593 if (should_check_uoh)
36595 for (int i = uoh_start_generation; i < total_generation_count && !should_collect; i++)
36597 should_collect = should_collect_optimized (hpt->dynamic_data_of (i), low_memory_p);
36600 #endif //MULTIPLE_HEAPS
36601 if (!should_collect)
36608 size_t CollectionCountAtEntry = dd_collection_count (dd);
36609 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
36610 size_t CurrentCollectionCount = 0;
36614 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
36616 if ((mode & collection_blocking) &&
36617 (generation == max_generation) &&
36618 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
36620 #ifdef BACKGROUND_GC
36621 if (gc_heap::background_running_p())
36623 pGenGCHeap->background_gc_wait();
36625 #endif //BACKGROUND_GC
36630 if (CollectionCountAtEntry == CurrentCollectionCount)
36639 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
36641 int gen = (generation < 0) ?
36642 max_generation : min (generation, max_generation);
36644 gc_reason reason = reason_empty;
36648 if (mode & collection_blocking)
36650 reason = reason_lowmemory_blocking;
36654 reason = reason_lowmemory;
36659 reason = reason_induced;
36662 if (reason == reason_induced)
36664 if (mode & collection_compacting)
36666 reason = reason_induced_compacting;
36668 else if (mode & collection_non_blocking)
36670 reason = reason_induced_noforce;
36673 else if (mode & collection_gcstress)
36675 reason = reason_gcstress;
36680 return GarbageCollectGeneration (gen, reason);
36683 #ifdef BACKGROUND_GC
36684 void gc_heap::add_bgc_pause_duration_0()
36686 if (settings.concurrent)
36688 uint64_t suspended_end_ts = GetHighPrecisionTimeStamp();
36689 size_t pause_duration = (size_t)(suspended_end_ts - suspended_start_time);
36690 last_recorded_gc_info* last_gc_info = &(last_bgc_info[last_bgc_info_index]);
36691 last_gc_info->pause_durations[0] = pause_duration;
36692 if (last_gc_info->index < last_ephemeral_gc_info.index)
36694 last_gc_info->pause_durations[0] -= last_ephemeral_gc_info.pause_durations[0];
36697 total_suspended_time += last_gc_info->pause_durations[0];
36701 last_recorded_gc_info* gc_heap::get_completed_bgc_info()
36703 int completed_bgc_index = gc_heap::background_running_p() ?
36704 (int)(!(gc_heap::last_bgc_info_index)) : (int)gc_heap::last_bgc_info_index;
36705 return &gc_heap::last_bgc_info[completed_bgc_index];
36707 #endif //BACKGROUND_GC
36709 void gc_heap::do_pre_gc()
36711 STRESS_LOG_GC_STACK;
36714 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
36715 (uint32_t)settings.condemned_generation,
36716 (uint32_t)settings.reason);
36717 #endif // STRESS_LOG
36719 #ifdef MULTIPLE_HEAPS
36720 gc_heap* hp = g_heaps[0];
36723 #endif //MULTIPLE_HEAPS
36725 #ifdef BACKGROUND_GC
36726 settings.b_state = hp->current_bgc_state;
36727 if (settings.concurrent)
36729 last_bgc_info_index = !last_bgc_info_index;
36730 last_bgc_info[last_bgc_info_index].index = settings.gc_index;
36732 #endif //BACKGROUND_GC
36735 size_t total_allocated_since_last_gc = get_total_allocated_since_last_gc();
36736 #ifdef BACKGROUND_GC
36737 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)(%s)(%d)",
36738 VolatileLoad(&settings.gc_index),
36739 dd_collection_count (hp->dynamic_data_of (0)),
36740 settings.condemned_generation,
36741 total_allocated_since_last_gc,
36742 (settings.concurrent ? "BGC" : (gc_heap::background_running_p() ? "FGC" : "NGC")),
36743 settings.b_state));
36745 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)",
36746 VolatileLoad(&settings.gc_index),
36747 dd_collection_count(hp->dynamic_data_of(0)),
36748 settings.condemned_generation,
36749 total_allocated_since_last_gc));
36750 #endif //BACKGROUND_GC
36752 if (heap_hard_limit)
36754 size_t total_heap_committed = get_total_committed_size();
36755 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
36756 dprintf (1, ("(%d)GC commit BEG #%Id: %Id (recorded: %Id)",
36757 settings.condemned_generation,
36758 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded));
36762 GCHeap::UpdatePreGCCounters();
36763 #if defined(__linux__)
36764 GCToEEInterface::UpdateGCEventStatus(static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Default)),
36765 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Default)),
36766 static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Private)),
36767 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Private)));
36768 #endif // __linux__
36770 if (settings.concurrent)
36772 #ifdef BACKGROUND_GC
36773 full_gc_counts[gc_type_background]++;
36774 #endif // BACKGROUND_GC
36778 if (settings.condemned_generation == max_generation)
36780 full_gc_counts[gc_type_blocking]++;
36784 #ifdef BACKGROUND_GC
36785 if (settings.background_p)
36787 ephemeral_fgc_counts[settings.condemned_generation]++;
36789 #endif //BACKGROUND_GC
36794 #ifdef GC_CONFIG_DRIVEN
36795 void gc_heap::record_interesting_info_per_heap()
36797 // datapoints are always from the last blocking GC so don't record again
36799 if (!(settings.concurrent))
36801 for (int i = 0; i < max_idp_count; i++)
36803 interesting_data_per_heap[i] += interesting_data_per_gc[i];
36807 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
36808 if (compact_reason >= 0)
36809 (compact_reasons_per_heap[compact_reason])++;
36810 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
36811 if (expand_mechanism >= 0)
36812 (expand_mechanisms_per_heap[expand_mechanism])++;
36814 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
36816 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
36817 (interesting_mechanism_bits_per_heap[i])++;
36820 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
36821 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
36823 (size_t)settings.gc_index,
36824 settings.condemned_generation,
36825 // TEMP - I am just doing this for wks GC 'cause I wanna see the pattern of doing C/S GCs.
36826 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
36827 ((expand_mechanism >= 0)? "X" : ""), // EX
36828 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
36829 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
36830 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
36831 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
36832 interesting_data_per_gc[idp_pre_short],
36833 interesting_data_per_gc[idp_post_short],
36834 interesting_data_per_gc[idp_merged_pin],
36835 interesting_data_per_gc[idp_converted_pin],
36836 interesting_data_per_gc[idp_pre_pin],
36837 interesting_data_per_gc[idp_post_pin],
36838 interesting_data_per_gc[idp_pre_and_post_pin],
36839 interesting_data_per_gc[idp_pre_short_padded],
36840 interesting_data_per_gc[idp_post_short_padded]));
36843 void gc_heap::record_global_mechanisms()
36845 for (int i = 0; i < max_global_mechanisms_count; i++)
36847 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
36849 ::record_global_mechanism (i);
36854 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
36856 if (!compact_ratio)
36857 return (!compact_p);
36859 size_t compact_count = compact_or_sweep_gcs[0];
36860 size_t sweep_count = compact_or_sweep_gcs[1];
36862 size_t total_count = compact_count + sweep_count;
36863 BOOL should_compact = compact_p;
36864 if (total_count > 3)
36868 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
36869 if (temp_ratio > compact_ratio)
36871 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
36872 // (compact_count + 1), (total_count + 1), temp_ratio));
36873 should_compact = FALSE;
36878 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
36879 if (temp_ratio > (100 - compact_ratio))
36881 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
36882 // (sweep_count + 1), (total_count + 1), temp_ratio));
36883 should_compact = TRUE;
36888 return !should_compact;
36890 #endif //GC_CONFIG_DRIVEN
36892 #ifdef BGC_SERVO_TUNING
36893 // virtual_fl_size is only used for NGC2
36894 void gc_heap::check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, ptrdiff_t virtual_fl_size)
36896 // For LOH we need to check more often to catch things like when the size grows too much.
36897 int min_gen_to_check = ((gen_number == max_generation) ? (max_generation - 1) : 0);
36899 if (settings.condemned_generation >= min_gen_to_check)
36901 #ifdef MULTIPLE_HEAPS
36902 gc_heap* hp = g_heaps[0];
36904 gc_heap* hp = pGenGCHeap;
36905 #endif //MULTIPLE_HEAPS
36907 size_t total_gen_size = physical_size;
36908 size_t total_generation_fl_size = get_total_generation_fl_size (gen_number);
36909 double gen_flr = (double)total_generation_fl_size * 100.0 / (double)total_gen_size;
36910 size_t gen1_index = dd_collection_count (hp->dynamic_data_of (max_generation - 1));
36911 size_t gen2_index = dd_collection_count (hp->dynamic_data_of (max_generation));
36913 bgc_tuning::tuning_calculation* current_gen_calc = &bgc_tuning::gen_calc[gen_number - max_generation];
36914 bgc_tuning::tuning_stats* current_gen_stats = &bgc_tuning::gen_stats[gen_number - max_generation];
36916 bool gen_size_inc_p = (total_gen_size > current_gen_calc->last_bgc_size);
36918 if ((settings.condemned_generation >= min_gen_to_check) &&
36919 (settings.condemned_generation != max_generation))
36921 if (gen_size_inc_p)
36923 current_gen_stats->last_gen_increase_flr = gen_flr;
36924 dprintf (BGC_TUNING_LOG, ("BTLp[g1: %Id, g2: %Id]: gen%d size inc %s %Id->%Id, flr: %.3f",
36925 gen1_index, gen2_index, gen_number,
36926 (gc_heap::background_running_p() ? "during bgc" : ""),
36927 current_gen_stats->last_bgc_physical_size, total_gen_size, gen_flr));
36930 if (!bgc_tuning::fl_tuning_triggered)
36932 if (bgc_tuning::enable_fl_tuning)
36934 if (!((gc_heap::background_running_p() || (hp->current_bgc_state == bgc_initialized))))
36936 assert (settings.entry_memory_load);
36938 // We start when we are 2/3 way there so we don't overshoot.
36939 if ((settings.entry_memory_load >= (bgc_tuning::memory_load_goal * 2 / 3)) &&
36940 (full_gc_counts[gc_type_background] >= 2))
36942 bgc_tuning::next_bgc_p = true;
36943 current_gen_calc->first_alloc_to_trigger = get_total_servo_alloc (gen_number);
36944 dprintf (BGC_TUNING_LOG, ("BTL[g1: %Id] mem high enough: %d(goal: %d), gen%d fl alloc: %Id, trigger BGC!",
36945 gen1_index, settings.entry_memory_load, bgc_tuning::memory_load_goal,
36946 gen_number, current_gen_calc->first_alloc_to_trigger));
36953 if ((settings.condemned_generation == max_generation) && !(settings.concurrent))
36955 size_t total_survived = get_total_surv_size (gen_number);
36956 size_t total_begin = get_total_begin_data_size (gen_number);
36957 double current_gc_surv_rate = (double)total_survived * 100.0 / (double)total_begin;
36959 // calculate the adjusted gen_flr.
36960 double total_virtual_size = (double)physical_size + (double)virtual_fl_size;
36961 double total_fl_size = (double)total_generation_fl_size + (double)virtual_fl_size;
36962 double new_gen_flr = total_fl_size * 100.0 / total_virtual_size;
36964 dprintf (BGC_TUNING_LOG, ("BTL%d NGC2 size %Id->%Id, fl %Id(%.3f)->%Id(%.3f)",
36965 gen_number, physical_size, (size_t)total_virtual_size,
36966 total_generation_fl_size, gen_flr,
36967 (size_t)total_fl_size, new_gen_flr));
36969 dprintf (BGC_TUNING_LOG, ("BTL%d* %Id, %.3f, %.3f, %.3f, %.3f, %.3f, %Id, %Id, %Id, %Id",
36971 (size_t)total_virtual_size,
36975 current_gen_stats->last_gen_increase_flr,
36976 current_gc_surv_rate,
36980 current_gen_calc->alloc_to_trigger));
36982 bgc_tuning::gen1_index_last_bgc_end = gen1_index;
36984 current_gen_calc->last_bgc_size = total_gen_size;
36985 current_gen_calc->last_bgc_flr = new_gen_flr;
36986 current_gen_calc->last_sweep_above_p = false;
36987 current_gen_calc->last_bgc_end_alloc = 0;
36989 current_gen_stats->last_alloc_end_to_start = 0;
36990 current_gen_stats->last_alloc_start_to_sweep = 0;
36991 current_gen_stats->last_alloc_sweep_to_end = 0;
36992 current_gen_stats->last_bgc_fl_size = total_generation_fl_size;
36993 current_gen_stats->last_bgc_surv_rate = current_gc_surv_rate;
36994 current_gen_stats->last_gen_increase_flr = 0;
36999 void gc_heap::get_and_reset_loh_alloc_info()
37001 if (!bgc_tuning::enable_fl_tuning)
37004 total_loh_a_last_bgc = 0;
37006 uint64_t total_loh_a_no_bgc = 0;
37007 uint64_t total_loh_a_bgc_marking = 0;
37008 uint64_t total_loh_a_bgc_planning = 0;
37009 #ifdef MULTIPLE_HEAPS
37010 for (int i = 0; i < gc_heap::n_heaps; i++)
37012 gc_heap* hp = gc_heap::g_heaps[i];
37013 #else //MULTIPLE_HEAPS
37015 gc_heap* hp = pGenGCHeap;
37016 #endif //MULTIPLE_HEAPS
37017 total_loh_a_no_bgc += hp->loh_a_no_bgc;
37018 hp->loh_a_no_bgc = 0;
37019 total_loh_a_bgc_marking += hp->loh_a_bgc_marking;
37020 hp->loh_a_bgc_marking = 0;
37021 total_loh_a_bgc_planning += hp->loh_a_bgc_planning;
37022 hp->loh_a_bgc_planning = 0;
37024 dprintf (2, ("LOH alloc: outside bgc: %I64d; bm: %I64d; bp: %I64d",
37025 total_loh_a_no_bgc,
37026 total_loh_a_bgc_marking,
37027 total_loh_a_bgc_planning));
37029 total_loh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning;
37031 #endif //BGC_SERVO_TUNING
37033 bool gc_heap::is_pm_ratio_exceeded()
37035 size_t maxgen_frag = 0;
37036 size_t maxgen_size = 0;
37037 size_t total_heap_size = get_total_heap_size();
37039 #ifdef MULTIPLE_HEAPS
37040 for (int i = 0; i < gc_heap::n_heaps; i++)
37042 gc_heap* hp = gc_heap::g_heaps[i];
37043 #else //MULTIPLE_HEAPS
37045 gc_heap* hp = pGenGCHeap;
37046 #endif //MULTIPLE_HEAPS
37048 maxgen_frag += dd_fragmentation (hp->dynamic_data_of (max_generation));
37049 maxgen_size += hp->generation_size (max_generation);
37052 double maxgen_ratio = (double)maxgen_size / (double)total_heap_size;
37053 double maxgen_frag_ratio = (double)maxgen_frag / (double)maxgen_size;
37054 dprintf (GTC_LOG, ("maxgen %Id(%d%% total heap), frag: %Id (%d%% maxgen)",
37055 maxgen_size, (int)(maxgen_ratio * 100.0),
37056 maxgen_frag, (int)(maxgen_frag_ratio * 100.0)));
37058 bool maxgen_highfrag_p = ((maxgen_ratio > 0.5) && (maxgen_frag_ratio > 0.1));
37060 // We need to adjust elevation here because if there's enough fragmentation it's not
37062 if (maxgen_highfrag_p)
37064 settings.should_lock_elevation = FALSE;
37065 dprintf (GTC_LOG, ("high frag gen2, turn off elevation"));
37068 return maxgen_highfrag_p;
37071 void gc_heap::update_recorded_gen_data (last_recorded_gc_info* gc_info)
37073 #ifdef MULTIPLE_HEAPS
37074 for (int i = 0; i < gc_heap::n_heaps; i++)
37076 gc_heap* hp = gc_heap::g_heaps[i];
37077 #else //MULTIPLE_HEAPS
37079 gc_heap* hp = pGenGCHeap;
37080 #endif //MULTIPLE_HEAPS
37082 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
37083 for (int gen_number = 0; gen_number < total_generation_count; gen_number++)
37085 recorded_generation_info* recorded_info = &(gc_info->gen_info[gen_number]);
37086 gc_generation_data* data = &(current_gc_data_per_heap->gen_data[gen_number]);
37087 recorded_info->size_before += data->size_before;
37088 recorded_info->fragmentation_before += data->free_list_space_before + data->free_obj_space_before;
37089 recorded_info->size_after += data->size_after;
37090 recorded_info->fragmentation_after += data->free_list_space_after + data->free_obj_space_after;
37095 void gc_heap::do_post_gc()
37097 if (!settings.concurrent)
37102 #ifdef MULTIPLE_HEAPS
37103 gc_heap* hp = g_heaps[0];
37106 #endif //MULTIPLE_HEAPS
37108 GCToEEInterface::GcDone(settings.condemned_generation);
37110 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
37111 (uint32_t)settings.condemned_generation,
37112 (uint32_t)settings.reason,
37113 !!settings.concurrent);
37117 uint32_t current_memory_load = 0;
37119 #ifdef BGC_SERVO_TUNING
37120 if (bgc_tuning::enable_fl_tuning)
37122 uint64_t current_available_physical = 0;
37123 size_t gen2_physical_size = 0;
37124 size_t gen3_physical_size = 0;
37125 ptrdiff_t gen2_virtual_fl_size = 0;
37126 ptrdiff_t gen3_virtual_fl_size = 0;
37127 ptrdiff_t vfl_from_kp = 0;
37128 ptrdiff_t vfl_from_ki = 0;
37130 gen2_physical_size = get_total_generation_size (max_generation);
37131 gen3_physical_size = get_total_generation_size (loh_generation);
37133 get_memory_info (¤t_memory_load, ¤t_available_physical);
37134 if ((settings.condemned_generation == max_generation) && !settings.concurrent)
37136 double gen2_size_ratio = (double)gen2_physical_size / ((double)gen2_physical_size + (double)gen3_physical_size);
37138 double total_virtual_fl_size = bgc_tuning::calculate_ml_tuning (current_available_physical, true, &vfl_from_kp, &vfl_from_ki);
37139 gen2_virtual_fl_size = (ptrdiff_t)(total_virtual_fl_size * gen2_size_ratio);
37140 gen3_virtual_fl_size = (ptrdiff_t)(total_virtual_fl_size * (1.0 - gen2_size_ratio));
37142 #ifdef SIMPLE_DPRINTF
37143 dprintf (BGC_TUNING_LOG, ("BTL: ml: %d (g: %d)(%s), a: %I64d (g: %I64d, elg: %Id+%Id=%Id, %Id+%Id=%Id), vfl: %Id=%Id+%Id(NGC2)",
37144 current_memory_load, bgc_tuning::memory_load_goal,
37145 ((current_available_physical > bgc_tuning::available_memory_goal) ? "above" : "below"),
37146 current_available_physical, bgc_tuning::available_memory_goal,
37147 gen2_physical_size, gen2_virtual_fl_size, (gen2_physical_size + gen2_virtual_fl_size),
37148 gen3_physical_size, gen3_virtual_fl_size, (gen3_physical_size + gen3_virtual_fl_size),
37149 (ptrdiff_t)total_virtual_fl_size, vfl_from_kp, vfl_from_ki));
37150 #endif //SIMPLE_DPRINTF
37153 check_and_adjust_bgc_tuning (max_generation, gen2_physical_size, gen2_virtual_fl_size);
37154 check_and_adjust_bgc_tuning (loh_generation, gen3_physical_size, gen3_virtual_fl_size);
37156 #endif //BGC_SERVO_TUNING
37158 #ifdef SIMPLE_DPRINTF
37159 dprintf (1, ("*EGC* %Id(gen0:%Id)(%Id)(%d)(%s)(%s)(%s)(ml: %d->%d)",
37160 VolatileLoad(&settings.gc_index),
37161 dd_collection_count(hp->dynamic_data_of(0)),
37162 (size_t)(GetHighPrecisionTimeStamp() / 1000),
37163 settings.condemned_generation,
37164 (settings.concurrent ? "BGC" : (gc_heap::background_running_p() ? "FGC" : "NGC")),
37165 (settings.compaction ? "C" : "S"),
37166 (settings.promotion ? "P" : "S"),
37167 settings.entry_memory_load,
37168 current_memory_load));
37169 #endif //SIMPLE_DPRINTF
37171 // Now record the gc info.
37172 last_recorded_gc_info* last_gc_info = 0;
37173 if (settings.concurrent)
37175 last_gc_info = &last_bgc_info[last_bgc_info_index];
37176 assert (last_gc_info->index == settings.gc_index);
37180 last_gc_info = ((settings.condemned_generation == max_generation) ?
37181 &last_full_blocking_gc_info : &last_ephemeral_gc_info);
37182 last_gc_info->index = settings.gc_index;
37184 size_t total_heap_committed = get_total_committed_size();
37185 last_gc_info->total_committed = total_heap_committed;
37186 last_gc_info->promoted = get_total_promoted();
37187 last_gc_info->pinned_objects = get_total_pinned_objects();
37188 last_gc_info->finalize_promoted_objects = GCHeap::GetFinalizablePromotedCount();
37190 if (!settings.concurrent)
37192 // If it's a normal blocking GC with its own SuspendEE, we simply get the elapsed time recoreded
37193 // and add the time between SuspendEE start and GC start.
37194 dynamic_data* dd = hp->dynamic_data_of (settings.condemned_generation);
37195 uint64_t gc_start_ts = dd_time_clock (dd);
37196 size_t pause_duration = (size_t)(end_gc_time - dd_time_clock (dd));
37198 if ((hp->current_bgc_state != bgc_initialized) && (settings.reason != reason_pm_full_gc))
37200 pause_duration += (size_t)(gc_start_ts - suspended_start_time);
37203 last_gc_info->pause_durations[0] = pause_duration;
37204 total_suspended_time += pause_duration;
37205 last_gc_info->pause_durations[1] = 0;
37208 uint64_t total_process_time = end_gc_time - process_start_time;
37209 last_gc_info->pause_percentage = (float)(total_process_time ?
37210 ((double)total_suspended_time / (double)total_process_time * 100.0) : 0);
37212 update_recorded_gen_data (last_gc_info);
37213 last_gc_info->heap_size = get_total_heap_size();
37214 last_gc_info->fragmentation = get_total_fragmentation();
37215 if (settings.exit_memory_load != 0)
37216 last_gc_info->memory_load = settings.exit_memory_load;
37217 else if (settings.entry_memory_load != 0)
37218 last_gc_info->memory_load = settings.entry_memory_load;
37219 last_gc_info->condemned_generation = settings.condemned_generation;
37220 last_gc_info->compaction = settings.compaction;
37221 last_gc_info->concurrent = settings.concurrent;
37223 #ifdef BACKGROUND_GC
37224 is_last_recorded_bgc = settings.concurrent;
37225 #endif //BACKGROUND_GC
37228 if (heap_hard_limit)
37230 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
37231 dprintf (1, ("(%d)GC commit END #%Id: %Id (recorded: %Id), heap %Id, frag: %Id",
37232 settings.condemned_generation,
37233 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded,
37234 last_gc_info->heap_size, last_gc_info->fragmentation));
37238 // Note we only do this at the end of full blocking GCs because we do not want
37239 // to turn on this provisional mode during the middle of a BGC.
37240 if ((settings.condemned_generation == max_generation) && (!settings.concurrent))
37244 size_t full_compacting_gc_count = full_gc_counts[gc_type_compacting];
37245 if (provisional_mode_triggered)
37247 uint64_t r = gc_rand::get_rand(10);
37248 if ((full_compacting_gc_count - provisional_triggered_gc_count) >= r)
37250 provisional_mode_triggered = false;
37251 provisional_off_gc_count = full_compacting_gc_count;
37252 dprintf (GTC_LOG, ("%Id NGC2s when turned on, %Id NGCs since(%Id)",
37253 provisional_triggered_gc_count, (full_compacting_gc_count - provisional_triggered_gc_count),
37254 num_provisional_triggered));
37259 uint64_t r = gc_rand::get_rand(5);
37260 if ((full_compacting_gc_count - provisional_off_gc_count) >= r)
37262 provisional_mode_triggered = true;
37263 provisional_triggered_gc_count = full_compacting_gc_count;
37264 num_provisional_triggered++;
37265 dprintf (GTC_LOG, ("%Id NGC2s when turned off, %Id NGCs since(%Id)",
37266 provisional_off_gc_count, (full_compacting_gc_count - provisional_off_gc_count),
37267 num_provisional_triggered));
37273 if (provisional_mode_triggered)
37275 if ((settings.entry_memory_load < high_memory_load_th) ||
37276 !is_pm_ratio_exceeded())
37278 dprintf (GTC_LOG, ("turning off PM"));
37279 provisional_mode_triggered = false;
37282 else if ((settings.entry_memory_load >= high_memory_load_th) && is_pm_ratio_exceeded())
37284 dprintf (GTC_LOG, ("highmem && highfrag - turning on PM"));
37285 provisional_mode_triggered = true;
37286 num_provisional_triggered++;
37291 GCHeap::UpdatePostGCCounters();
37293 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
37294 (uint32_t)settings.condemned_generation,
37295 (uint32_t)settings.reason);
37296 #endif // STRESS_LOG
37298 #ifdef GC_CONFIG_DRIVEN
37299 if (!settings.concurrent)
37301 if (settings.compaction)
37302 (compact_or_sweep_gcs[0])++;
37304 (compact_or_sweep_gcs[1])++;
37307 #ifdef MULTIPLE_HEAPS
37308 for (int i = 0; i < n_heaps; i++)
37309 g_heaps[i]->record_interesting_info_per_heap();
37311 record_interesting_info_per_heap();
37312 #endif //MULTIPLE_HEAPS
37313 if (mark_list_overflow)
37316 mark_list_overflow = false;
37319 record_global_mechanisms();
37320 #endif //GC_CONFIG_DRIVEN
37323 unsigned GCHeap::GetGcCount()
37325 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
37329 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
37331 dprintf (2, ("triggered a GC!"));
37333 #ifdef MULTIPLE_HEAPS
37334 gc_heap* hpt = gc_heap::g_heaps[0];
37337 #endif //MULTIPLE_HEAPS
37338 bool cooperative_mode = true;
37339 dynamic_data* dd = hpt->dynamic_data_of (gen);
37340 size_t localCount = dd_collection_count (dd);
37342 enter_spin_lock (&gc_heap::gc_lock);
37343 dprintf (SPINLOCK_LOG, ("GC Egc"));
37344 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
37346 //don't trigger another GC if one was already in progress
37347 //while waiting for the lock
37349 size_t col_count = dd_collection_count (dd);
37351 if (localCount != col_count)
37353 #ifdef SYNCHRONIZATION_STATS
37354 gc_lock_contended++;
37355 #endif //SYNCHRONIZATION_STATS
37356 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
37357 leave_spin_lock (&gc_heap::gc_lock);
37359 // We don't need to release msl here 'cause this means a GC
37360 // has happened and would have release all msl's.
37365 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
37366 (reason == reason_lowmemory_blocking) ||
37367 (gc_heap::latency_level == latency_level_memory_footprint);
37369 gc_trigger_reason = reason;
37371 #ifdef MULTIPLE_HEAPS
37372 for (int i = 0; i < gc_heap::n_heaps; i++)
37374 gc_heap::g_heaps[i]->reset_gc_done();
37377 gc_heap::reset_gc_done();
37378 #endif //MULTIPLE_HEAPS
37380 gc_heap::gc_started = TRUE;
37383 init_sync_log_stats();
37385 #ifndef MULTIPLE_HEAPS
37386 cooperative_mode = gc_heap::enable_preemptive ();
37388 dprintf (2, ("Suspending EE"));
37389 gc_heap::suspended_start_time = GetHighPrecisionTimeStamp();
37390 BEGIN_TIMING(suspend_ee_during_log);
37391 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
37392 END_TIMING(suspend_ee_during_log);
37393 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
37394 gc_heap::disable_preemptive (cooperative_mode);
37395 if (gc_heap::proceed_with_gc_p)
37396 pGenGCHeap->settings.init_mechanisms();
37398 gc_heap::update_collection_counts_for_no_gc();
37400 #endif //!MULTIPLE_HEAPS
37403 unsigned int condemned_generation_number = gen;
37405 // We want to get a stack from the user thread that triggered the GC
37406 // instead of on the GC thread which is the case for Server GC.
37407 // But we are doing it for Workstation GC as well to be uniform.
37408 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
37410 #ifdef MULTIPLE_HEAPS
37411 GcCondemnedGeneration = condemned_generation_number;
37413 cooperative_mode = gc_heap::enable_preemptive ();
37415 BEGIN_TIMING(gc_during_log);
37416 gc_heap::ee_suspend_event.Set();
37417 gc_heap::wait_for_gc_done();
37418 END_TIMING(gc_during_log);
37420 gc_heap::disable_preemptive (cooperative_mode);
37422 condemned_generation_number = GcCondemnedGeneration;
37424 if (gc_heap::proceed_with_gc_p)
37426 BEGIN_TIMING(gc_during_log);
37427 pGenGCHeap->garbage_collect (condemned_generation_number);
37428 if (gc_heap::pm_trigger_full_gc)
37430 pGenGCHeap->garbage_collect_pm_full_gc();
37432 END_TIMING(gc_during_log);
37434 #endif //MULTIPLE_HEAPS
37436 #ifdef BACKGROUND_GC
37437 // We are deciding whether we should fire the alloc wait end event here
37438 // because in begin_foreground we could be calling end_foreground
37439 // if we need to retry.
37440 if (gc_heap::alloc_wait_event_p)
37442 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
37443 gc_heap::alloc_wait_event_p = FALSE;
37445 #endif //BACKGROUND_GC
37447 #ifndef MULTIPLE_HEAPS
37448 #ifdef BACKGROUND_GC
37449 if (!gc_heap::dont_restart_ee_p)
37450 #endif //BACKGROUND_GC
37452 #ifdef BACKGROUND_GC
37453 gc_heap::add_bgc_pause_duration_0();
37454 #endif //BACKGROUND_GC
37455 BEGIN_TIMING(restart_ee_during_log);
37456 GCToEEInterface::RestartEE(TRUE);
37457 END_TIMING(restart_ee_during_log);
37459 #endif //!MULTIPLE_HEAPS
37461 #ifndef MULTIPLE_HEAPS
37462 process_sync_log_stats();
37463 gc_heap::gc_started = FALSE;
37464 gc_heap::set_gc_done();
37465 dprintf (SPINLOCK_LOG, ("GC Lgc"));
37466 leave_spin_lock (&gc_heap::gc_lock);
37467 #endif //!MULTIPLE_HEAPS
37469 #ifdef FEATURE_PREMORTEM_FINALIZATION
37470 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
37471 #endif // FEATURE_PREMORTEM_FINALIZATION
37473 return dd_collection_count (dd);
37476 size_t GCHeap::GetTotalBytesInUse ()
37478 #ifdef MULTIPLE_HEAPS
37479 //enumerate all the heaps and get their size.
37480 size_t tot_size = 0;
37481 for (int i = 0; i < gc_heap::n_heaps; i++)
37483 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
37484 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
37488 return ApproxTotalBytesInUse ();
37489 #endif //MULTIPLE_HEAPS
37492 // Get the total allocated bytes
37493 uint64_t GCHeap::GetTotalAllocatedBytes()
37495 #ifdef MULTIPLE_HEAPS
37496 uint64_t total_alloc_bytes = 0;
37497 for (int i = 0; i < gc_heap::n_heaps; i++)
37499 gc_heap* hp = gc_heap::g_heaps[i];
37500 total_alloc_bytes += hp->total_alloc_bytes_soh;
37501 total_alloc_bytes += hp->total_alloc_bytes_uoh;
37503 return total_alloc_bytes;
37505 return (pGenGCHeap->total_alloc_bytes_soh + pGenGCHeap->total_alloc_bytes_uoh);
37506 #endif //MULTIPLE_HEAPS
37509 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
37511 if (get_bgc_fgc_count != 0)
37513 #ifdef BACKGROUND_GC
37514 if (generation == max_generation)
37516 return (int)(gc_heap::full_gc_counts[gc_type_background]);
37520 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
37524 #endif //BACKGROUND_GC
37527 #ifdef MULTIPLE_HEAPS
37528 gc_heap* hp = gc_heap::g_heaps [0];
37529 #else //MULTIPLE_HEAPS
37530 gc_heap* hp = pGenGCHeap;
37531 #endif //MULTIPLE_HEAPS
37532 if (generation > max_generation)
37535 return (int)dd_collection_count (hp->dynamic_data_of (generation));
37538 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
37540 size_t totsize = 0;
37541 enter_spin_lock (&pGenGCHeap->gc_lock);
37543 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
37544 // Get small block heap size info
37545 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
37546 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
37547 while (seg1 != eph_seg)
37549 totsize += heap_segment_allocated (seg1) -
37550 heap_segment_mem (seg1);
37551 seg1 = heap_segment_next (seg1);
37554 //discount the fragmentation
37555 for (int i = 0; i <= max_generation; i++)
37557 generation* gen = pGenGCHeap->generation_of (i);
37558 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
37561 if (!small_heap_only)
37563 for (int i = uoh_start_generation; i < total_generation_count; i++)
37565 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (i));
37569 totsize += heap_segment_allocated (seg2) -
37570 heap_segment_mem (seg2);
37571 seg2 = heap_segment_next (seg2);
37574 //discount the fragmentation
37575 generation* uoh_gen = pGenGCHeap->generation_of (i);
37576 size_t frag = generation_free_list_space (uoh_gen) + generation_free_obj_space (uoh_gen);
37580 leave_spin_lock (&pGenGCHeap->gc_lock);
37584 #ifdef MULTIPLE_HEAPS
37585 void GCHeap::AssignHeap (alloc_context* acontext)
37587 // Assign heap based on processor
37588 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext)));
37589 acontext->set_home_heap(acontext->get_alloc_heap());
37592 GCHeap* GCHeap::GetHeap (int n)
37594 assert (n < gc_heap::n_heaps);
37595 return gc_heap::g_heaps[n]->vm_heap;
37597 #endif //MULTIPLE_HEAPS
37599 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
37601 alloc_context* acontext = static_cast<alloc_context*>(context);
37602 #ifdef MULTIPLE_HEAPS
37603 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
37604 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
37606 UNREFERENCED_PARAMETER(acontext);
37607 UNREFERENCED_PARAMETER(thread_number);
37609 #endif //MULTIPLE_HEAPS
37612 // Returns the number of processors required to trigger the use of thread based allocation contexts
37613 int GCHeap::GetNumberOfHeaps ()
37615 #ifdef MULTIPLE_HEAPS
37616 return gc_heap::n_heaps;
37619 #endif //MULTIPLE_HEAPS
37623 in this way we spend extra time cycling through all the heaps while create the handle
37624 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
37626 int GCHeap::GetHomeHeapNumber ()
37628 #ifdef MULTIPLE_HEAPS
37629 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
37635 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
37636 return (hp ? hp->pGenGCHeap->heap_number : 0);
37639 #endif //MULTIPLE_HEAPS
37642 unsigned int GCHeap::GetCondemnedGeneration()
37644 return gc_heap::settings.condemned_generation;
37647 void GCHeap::GetMemoryInfo(uint64_t* highMemLoadThresholdBytes,
37648 uint64_t* totalAvailableMemoryBytes,
37649 uint64_t* lastRecordedMemLoadBytes,
37650 uint64_t* lastRecordedHeapSizeBytes,
37651 uint64_t* lastRecordedFragmentationBytes,
37652 uint64_t* totalCommittedBytes,
37653 uint64_t* promotedBytes,
37654 uint64_t* pinnedObjectCount,
37655 uint64_t* finalizationPendingCount,
37657 uint32_t* generation,
37658 uint32_t* pauseTimePct,
37659 bool* isCompaction,
37660 bool* isConcurrent,
37661 uint64_t* genInfoRaw,
37662 uint64_t* pauseInfoRaw,
37665 last_recorded_gc_info* last_gc_info = 0;
37667 if ((gc_kind)kind == gc_kind_ephemeral)
37669 last_gc_info = &gc_heap::last_ephemeral_gc_info;
37671 else if ((gc_kind)kind == gc_kind_full_blocking)
37673 last_gc_info = &gc_heap::last_full_blocking_gc_info;
37675 #ifdef BACKGROUND_GC
37676 else if ((gc_kind)kind == gc_kind_background)
37678 last_gc_info = gc_heap::get_completed_bgc_info();
37680 #endif //BACKGROUND_GC
37683 assert ((gc_kind)kind == gc_kind_any);
37684 #ifdef BACKGROUND_GC
37685 if (gc_heap::is_last_recorded_bgc)
37687 last_gc_info = gc_heap::get_completed_bgc_info();
37690 #endif //BACKGROUND_GC
37692 last_gc_info = ((gc_heap::last_ephemeral_gc_info.index > gc_heap::last_full_blocking_gc_info.index) ?
37693 &gc_heap::last_ephemeral_gc_info : &gc_heap::last_full_blocking_gc_info);
37697 *highMemLoadThresholdBytes = (uint64_t) (((double)(gc_heap::high_memory_load_th)) / 100 * gc_heap::total_physical_mem);
37698 *totalAvailableMemoryBytes = gc_heap::heap_hard_limit != 0 ? gc_heap::heap_hard_limit : gc_heap::total_physical_mem;
37699 *lastRecordedMemLoadBytes = (uint64_t) (((double)(last_gc_info->memory_load)) / 100 * gc_heap::total_physical_mem);
37700 *lastRecordedHeapSizeBytes = last_gc_info->heap_size;
37701 *lastRecordedFragmentationBytes = last_gc_info->fragmentation;
37702 *totalCommittedBytes = last_gc_info->total_committed;
37703 *promotedBytes = last_gc_info->promoted;
37704 *pinnedObjectCount = last_gc_info->pinned_objects;
37705 *finalizationPendingCount = last_gc_info->finalize_promoted_objects;
37706 *index = last_gc_info->index;
37707 *generation = last_gc_info->condemned_generation;
37708 *pauseTimePct = (int)(last_gc_info->pause_percentage * 100);
37709 *isCompaction = last_gc_info->compaction;
37710 *isConcurrent = last_gc_info->concurrent;
37711 int genInfoIndex = 0;
37712 for (int i = 0; i < total_generation_count; i++)
37714 genInfoRaw[genInfoIndex++] = last_gc_info->gen_info[i].size_before;
37715 genInfoRaw[genInfoIndex++] = last_gc_info->gen_info[i].fragmentation_before;
37716 genInfoRaw[genInfoIndex++] = last_gc_info->gen_info[i].size_after;
37717 genInfoRaw[genInfoIndex++] = last_gc_info->gen_info[i].fragmentation_after;
37719 for (int i = 0; i < 2; i++)
37721 // convert it to 100-ns units that TimeSpan needs.
37722 pauseInfoRaw[i] = (uint64_t)(last_gc_info->pause_durations[i]) * 10;
37726 if ((gc_kind)kind == gc_kind_ephemeral)
37728 assert (last_gc_info->condemned_generation < max_generation);
37730 else if ((gc_kind)kind == gc_kind_full_blocking)
37732 assert (last_gc_info->condemned_generation == max_generation);
37733 assert (last_gc_info->concurrent == false);
37735 #ifdef BACKGROUND_GC
37736 else if ((gc_kind)kind == gc_kind_background)
37738 assert (last_gc_info->condemned_generation == max_generation);
37739 assert (last_gc_info->concurrent == true);
37741 #endif //BACKGROUND_GC
37745 uint32_t GCHeap::GetMemoryLoad()
37747 uint32_t memory_load = 0;
37748 if (gc_heap::settings.exit_memory_load != 0)
37749 memory_load = gc_heap::settings.exit_memory_load;
37750 else if (gc_heap::settings.entry_memory_load != 0)
37751 memory_load = gc_heap::settings.entry_memory_load;
37753 return memory_load;
37756 int GCHeap::GetGcLatencyMode()
37758 return (int)(pGenGCHeap->settings.pause_mode);
37761 int GCHeap::SetGcLatencyMode (int newLatencyMode)
37763 if (gc_heap::settings.pause_mode == pause_no_gc)
37764 return (int)set_pause_mode_no_gc;
37766 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
37768 if (new_mode == pause_low_latency)
37770 #ifndef MULTIPLE_HEAPS
37771 pGenGCHeap->settings.pause_mode = new_mode;
37772 #endif //!MULTIPLE_HEAPS
37774 else if (new_mode == pause_sustained_low_latency)
37776 #ifdef BACKGROUND_GC
37777 if (gc_heap::gc_can_use_concurrent)
37779 pGenGCHeap->settings.pause_mode = new_mode;
37781 #endif //BACKGROUND_GC
37785 pGenGCHeap->settings.pause_mode = new_mode;
37788 #ifdef BACKGROUND_GC
37789 if (gc_heap::background_running_p())
37791 // If we get here, it means we are doing an FGC. If the pause
37792 // mode was altered we will need to save it in the BGC settings.
37793 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
37795 gc_heap::saved_bgc_settings.pause_mode = new_mode;
37798 #endif //BACKGROUND_GC
37800 return (int)set_pause_mode_success;
37803 int GCHeap::GetLOHCompactionMode()
37805 return pGenGCHeap->loh_compaction_mode;
37808 void GCHeap::SetLOHCompactionMode (int newLOHCompactionMode)
37810 #ifdef FEATURE_LOH_COMPACTION
37811 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionMode;
37812 #endif //FEATURE_LOH_COMPACTION
37815 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
37816 uint32_t lohPercentage)
37818 #ifdef MULTIPLE_HEAPS
37819 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37821 gc_heap* hp = gc_heap::g_heaps [hn];
37822 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
37823 hp->fgn_maxgen_percent = gen2Percentage;
37825 #else //MULTIPLE_HEAPS
37826 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
37827 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
37828 #endif //MULTIPLE_HEAPS
37830 pGenGCHeap->full_gc_approach_event.Reset();
37831 pGenGCHeap->full_gc_end_event.Reset();
37832 pGenGCHeap->full_gc_approach_event_set = false;
37834 pGenGCHeap->fgn_loh_percent = lohPercentage;
37839 bool GCHeap::CancelFullGCNotification()
37841 #ifdef MULTIPLE_HEAPS
37842 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37844 gc_heap* hp = gc_heap::g_heaps [hn];
37845 hp->fgn_maxgen_percent = 0;
37847 #else //MULTIPLE_HEAPS
37848 pGenGCHeap->fgn_maxgen_percent = 0;
37849 #endif //MULTIPLE_HEAPS
37851 pGenGCHeap->fgn_loh_percent = 0;
37852 pGenGCHeap->full_gc_approach_event.Set();
37853 pGenGCHeap->full_gc_end_event.Set();
37858 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
37860 dprintf (2, ("WFGA: Begin wait"));
37861 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
37862 dprintf (2, ("WFGA: End wait"));
37866 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
37868 dprintf (2, ("WFGE: Begin wait"));
37869 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
37870 dprintf (2, ("WFGE: End wait"));
37874 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
37876 NoGCRegionLockHolder lh;
37878 dprintf (1, ("begin no gc called"));
37879 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
37880 if (status == start_no_gc_success)
37882 GarbageCollect (max_generation);
37883 status = gc_heap::get_start_no_gc_region_status();
37886 if (status != start_no_gc_success)
37887 gc_heap::handle_failure_for_no_gc();
37889 return (int)status;
37892 int GCHeap::EndNoGCRegion()
37894 NoGCRegionLockHolder lh;
37895 return (int)gc_heap::end_no_gc_region();
37898 void GCHeap::PublishObject (uint8_t* Obj)
37900 #ifdef BACKGROUND_GC
37901 gc_heap* hp = gc_heap::heap_of (Obj);
37902 hp->bgc_alloc_lock->uoh_alloc_done (Obj);
37903 hp->bgc_untrack_uoh_alloc();
37904 #endif //BACKGROUND_GC
37907 // The spec for this one isn't clear. This function
37908 // returns the size that can be allocated without
37909 // triggering a GC of any kind.
37910 size_t GCHeap::ApproxFreeBytes()
37912 enter_spin_lock (&pGenGCHeap->gc_lock);
37914 generation* gen = pGenGCHeap->generation_of (0);
37915 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
37917 leave_spin_lock (&pGenGCHeap->gc_lock);
37922 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
37924 if ((gen < 0) || (gen > max_generation))
37926 #ifdef MULTIPLE_HEAPS
37927 counters->current_size = 0;
37928 counters->promoted_size = 0;
37929 counters->collection_count = 0;
37931 //enumerate all the heaps and get their counters.
37932 for (int i = 0; i < gc_heap::n_heaps; i++)
37934 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
37936 counters->current_size += dd_current_size (dd);
37937 counters->promoted_size += dd_promoted_size (dd);
37939 counters->collection_count += dd_collection_count (dd);
37942 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
37943 counters->current_size = dd_current_size (dd);
37944 counters->promoted_size = dd_promoted_size (dd);
37945 counters->collection_count = dd_collection_count (dd);
37946 #endif //MULTIPLE_HEAPS
37950 // Get the segment size to use, making sure it conforms.
37951 size_t GCHeap::GetValidSegmentSize(bool large_seg)
37953 return (large_seg ? gc_heap::min_uoh_segment_size : gc_heap::soh_segment_size);
37956 size_t gc_heap::get_gen0_min_size()
37958 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
37959 bool is_config_invalid = ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size));
37960 if (is_config_invalid)
37963 // performance data seems to indicate halving the size results
37964 // in optimal perf. Ask for adjusted gen0 size.
37965 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
37967 // if gen0 size is too large given the available memory, reduce it.
37968 // Get true cache size, as we don't want to reduce below this.
37969 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
37970 dprintf (1, ("cache: %Id-%Id",
37971 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
37972 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
37974 int n_heaps = gc_heap::n_heaps;
37976 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
37977 gen0size = max((4*trueSize/5),(256*1024));
37978 trueSize = max(trueSize, (256*1024));
37982 dprintf (1, ("gen0size: %Id * %d = %Id, physical mem: %Id / 6 = %Id",
37983 gen0size, n_heaps, (gen0size * n_heaps),
37984 gc_heap::total_physical_mem,
37985 gc_heap::total_physical_mem / 6));
37987 // if the total min GC across heaps will exceed 1/6th of available memory,
37988 // then reduce the min GC size until it either fits or has been reduced to cache size.
37989 while ((gen0size * n_heaps) > (gc_heap::total_physical_mem / 6))
37991 gen0size = gen0size / 2;
37992 if (gen0size <= trueSize)
37994 gen0size = trueSize;
38000 size_t seg_size = gc_heap::soh_segment_size;
38003 // Generation 0 must never be more than 1/2 the segment size.
38004 if (gen0size >= (seg_size / 2))
38005 gen0size = seg_size / 2;
38007 // If the value from config is valid we use it as is without this adjustment.
38008 if (is_config_invalid)
38010 if (heap_hard_limit)
38012 size_t gen0size_seg = seg_size / 8;
38013 if (gen0size >= gen0size_seg)
38015 dprintf (1, ("gen0 limited by seg size %Id->%Id", gen0size, gen0size_seg));
38016 gen0size = gen0size_seg;
38020 gen0size = gen0size / 8 * 5;
38023 gen0size = Align (gen0size);
38028 void GCHeap::SetReservedVMLimit (size_t vmlimit)
38030 gc_heap::reserved_memory_limit = vmlimit;
38033 //versions of same method on each heap
38035 #ifdef FEATURE_PREMORTEM_FINALIZATION
38037 Object* GCHeap::GetNextFinalizableObject()
38040 #ifdef MULTIPLE_HEAPS
38042 //return the first non critical one in the first queue.
38043 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38045 gc_heap* hp = gc_heap::g_heaps [hn];
38046 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
38050 //return the first non critical/critical one in the first queue.
38051 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38053 gc_heap* hp = gc_heap::g_heaps [hn];
38054 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
38061 #else //MULTIPLE_HEAPS
38062 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
38063 #endif //MULTIPLE_HEAPS
38067 size_t GCHeap::GetNumberFinalizableObjects()
38069 #ifdef MULTIPLE_HEAPS
38071 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38073 gc_heap* hp = gc_heap::g_heaps [hn];
38074 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
38079 #else //MULTIPLE_HEAPS
38080 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
38081 #endif //MULTIPLE_HEAPS
38084 size_t GCHeap::GetFinalizablePromotedCount()
38086 #ifdef MULTIPLE_HEAPS
38089 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38091 gc_heap* hp = gc_heap::g_heaps [hn];
38092 cnt += hp->finalize_queue->GetPromotedCount();
38096 #else //MULTIPLE_HEAPS
38097 return pGenGCHeap->finalize_queue->GetPromotedCount();
38098 #endif //MULTIPLE_HEAPS
38101 //---------------------------------------------------------------------------
38102 // Finalized class tracking
38103 //---------------------------------------------------------------------------
38105 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
38109 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
38111 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
38116 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
38117 return hp->finalize_queue->RegisterForFinalization (gen, obj);
38121 void GCHeap::SetFinalizationRun (Object* obj)
38123 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
38127 //--------------------------------------------------------------------
38129 // Support for finalization
38131 //--------------------------------------------------------------------
38134 unsigned int gen_segment (int gen)
38136 assert (((signed)total_generation_count - gen - 1)>=0);
38137 return (total_generation_count - gen - 1);
38140 bool CFinalize::Initialize()
38147 m_Array = new (nothrow)(Object*[100]);
38152 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
38153 if (GCConfig::GetBreakOnOOM())
38155 GCToOSInterface::DebugBreak();
38159 m_EndArray = &m_Array[100];
38161 for (int i =0; i < FreeList; i++)
38163 SegQueueLimit (i) = m_Array;
38165 m_PromotedCount = 0;
38168 lockowner_threadid.Clear();
38174 CFinalize::~CFinalize()
38179 size_t CFinalize::GetPromotedCount ()
38181 return m_PromotedCount;
38185 void CFinalize::EnterFinalizeLock()
38187 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
38188 GCToEEInterface::GetThread() == 0 ||
38189 GCToEEInterface::IsPreemptiveGCDisabled());
38192 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
38194 unsigned int i = 0;
38197 YieldProcessor(); // indicate to the processor that we are spinning
38199 GCToOSInterface::YieldThread (0);
38201 GCToOSInterface::Sleep (5);
38207 lockowner_threadid.SetToCurrentThread();
38212 void CFinalize::LeaveFinalizeLock()
38214 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
38215 GCToEEInterface::GetThread() == 0 ||
38216 GCToEEInterface::IsPreemptiveGCDisabled());
38219 lockowner_threadid.Clear();
38225 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
38232 EnterFinalizeLock();
38235 unsigned int dest = gen_segment (gen);
38237 // Adjust boundary for segments so that GC will keep objects alive.
38238 Object*** s_i = &SegQueue (FreeList);
38239 if ((*s_i) == m_EndArray)
38243 LeaveFinalizeLock();
38244 if (method_table(obj) == NULL)
38246 // If the object is uninitialized, a valid size should have been passed.
38247 assert (size >= Align (min_obj_size));
38248 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
38249 ((CObjectHeader*)obj)->SetFree(size);
38251 STRESS_LOG_OOM_STACK(0);
38252 if (GCConfig::GetBreakOnOOM())
38254 GCToOSInterface::DebugBreak();
38259 Object*** end_si = &SegQueueLimit (dest);
38262 //is the segment empty?
38263 if (!(*s_i == *(s_i-1)))
38265 //no, swap the end elements.
38266 *(*s_i) = *(*(s_i-1));
38268 //increment the fill pointer
38270 //go to the next segment.
38272 } while (s_i > end_si);
38274 // We have reached the destination segment
38275 // store the object
38277 // increment the fill pointer
38280 LeaveFinalizeLock();
38286 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
38289 EnterFinalizeLock();
38291 if (!IsSegEmpty(FinalizerListSeg))
38293 obj = *(--SegQueueLimit (FinalizerListSeg));
38295 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
38297 //the FinalizerList is empty, we can adjust both
38298 // limit instead of moving the object to the free list
38299 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
38300 --SegQueueLimit (FinalizerListSeg);
38304 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
38306 LeaveFinalizeLock();
38311 CFinalize::GetNumberFinalizableObjects()
38313 return SegQueueLimit(FinalizerListSeg) - SegQueue(FinalizerListSeg);
38317 CFinalize::MoveItem (Object** fromIndex,
38318 unsigned int fromSeg,
38319 unsigned int toSeg)
38323 ASSERT (fromSeg != toSeg);
38324 if (fromSeg > toSeg)
38328 // Place the element at the boundary closest to dest
38329 Object** srcIndex = fromIndex;
38330 for (unsigned int i = fromSeg; i != toSeg; i+= step)
38332 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
38333 Object** destIndex = destFill - (step + 1)/2;
38334 if (srcIndex != destIndex)
38336 Object* tmp = *srcIndex;
38337 *srcIndex = *destIndex;
38341 srcIndex = destIndex;
38346 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
38352 pSC->thread_number = hn;
38354 //scan the finalization queue
38355 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
38356 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
38358 for (Object** po = startIndex; po < stopIndex; po++)
38361 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
38362 dprintf (3, ("scan f %Ix", (size_t)o));
38368 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
38370 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
38371 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
38372 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
38373 for (Object** po = startIndex; po < stopIndex; po++)
38375 fn(po < stopCriticalIndex, *po);
38380 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
38384 sc.promotion = TRUE;
38385 #ifdef MULTIPLE_HEAPS
38386 sc.thread_number = hp->heap_number;
38388 UNREFERENCED_PARAMETER(hp);
38389 #endif //MULTIPLE_HEAPS
38391 BOOL finalizedFound = FALSE;
38393 //start with gen and explore all the younger generations.
38394 unsigned int startSeg = gen_segment (gen);
38396 m_PromotedCount = 0;
38397 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
38399 Object** endIndex = SegQueue (Seg);
38400 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
38402 CObjectHeader* obj = (CObjectHeader*)*i;
38403 dprintf (3, ("scanning: %Ix", (size_t)obj));
38404 if (!g_theGCHeap->IsPromoted (obj))
38406 dprintf (3, ("freacheable: %Ix", (size_t)obj));
38408 assert (method_table(obj)->HasFinalizer());
38410 if (GCToEEInterface::EagerFinalized(obj))
38412 MoveItem (i, Seg, FreeList);
38414 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
38416 //remove the object because we don't want to
38417 //run the finalizer
38418 MoveItem (i, Seg, FreeList);
38420 //Reset the bit so it will be put back on the queue
38421 //if resurrected and re-registered.
38422 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
38429 if (method_table(obj)->HasCriticalFinalizer())
38431 MoveItem (i, Seg, CriticalFinalizerListSeg);
38435 MoveItem (i, Seg, FinalizerListSeg);
38439 #ifdef BACKGROUND_GC
38442 if ((gen == max_generation) && (gc_heap::background_running_p()))
38444 // TODO - fix the following line.
38445 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
38446 dprintf (3, ("%Ix is marked", (size_t)obj));
38449 #endif //BACKGROUND_GC
38453 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
38454 !IsSegEmpty(CriticalFinalizerListSeg);
38456 if (finalizedFound)
38458 //Promote the f-reachable objects
38460 #ifdef MULTIPLE_HEAPS
38464 #endif //MULTIPLE_HEAPS
38467 hp->settings.found_finalizers = TRUE;
38469 #ifdef BACKGROUND_GC
38470 if (hp->settings.concurrent)
38472 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
38474 #endif //BACKGROUND_GC
38475 if (hp->settings.concurrent && hp->settings.found_finalizers)
38478 GCToEEInterface::EnableFinalization(true);
38482 return finalizedFound;
38485 //Relocates all of the objects in the finalization array
38487 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
38490 sc.promotion = FALSE;
38491 #ifdef MULTIPLE_HEAPS
38492 sc.thread_number = hp->heap_number;
38494 UNREFERENCED_PARAMETER(hp);
38495 #endif //MULTIPLE_HEAPS
38497 unsigned int Seg = gen_segment (gen);
38499 Object** startIndex = SegQueue (Seg);
38500 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
38502 GCHeap::Relocate (po, &sc);
38507 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
38509 // update the generation fill pointers.
38510 // if gen_0_empty is FALSE, test each object to find out if
38511 // it was promoted or not
38514 for (int i = min (gen+1, max_generation); i > 0; i--)
38516 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
38521 //Look for demoted or promoted objects
38522 for (int i = gen; i >= 0; i--)
38524 unsigned int Seg = gen_segment (i);
38525 Object** startIndex = SegQueue (Seg);
38527 for (Object** po = startIndex;
38528 po < SegQueueLimit (gen_segment(i)); po++)
38530 int new_gen = g_theGCHeap->WhichGeneration (*po);
38536 MoveItem (po, gen_segment (i), gen_segment (new_gen));
38541 MoveItem (po, gen_segment (i), gen_segment (new_gen));
38542 //back down in order to see all objects.
38552 CFinalize::GrowArray()
38554 size_t oldArraySize = (m_EndArray - m_Array);
38555 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
38557 Object** newArray = new (nothrow) Object*[newArraySize];
38562 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
38564 //adjust the fill pointers
38565 for (int i = 0; i < FreeList; i++)
38567 m_FillPointers [i] += (newArray - m_Array);
38570 m_Array = newArray;
38571 m_EndArray = &m_Array [newArraySize];
38577 void CFinalize::CheckFinalizerObjects()
38579 for (int i = 0; i <= max_generation; i++)
38581 Object **startIndex = SegQueue (gen_segment (i));
38582 Object **stopIndex = SegQueueLimit (gen_segment (i));
38584 for (Object **po = startIndex; po < stopIndex; po++)
38586 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
38588 ((CObjectHeader*)*po)->Validate();
38592 #endif //VERIFY_HEAP
38594 #endif // FEATURE_PREMORTEM_FINALIZATION
38597 //------------------------------------------------------------------------------
38599 // End of VM specific support
38601 //------------------------------------------------------------------------------
38602 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
38604 generation* gen = gc_heap::generation_of (gen_number);
38605 heap_segment* seg = generation_start_segment (gen);
38606 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
38607 generation_allocation_start (gen));
38609 uint8_t* end = heap_segment_allocated (seg);
38610 BOOL small_object_segments = TRUE;
38611 int align_const = get_alignment_constant (small_object_segments);
38618 if ((seg = heap_segment_next (seg)) != 0)
38620 x = heap_segment_mem (seg);
38621 end = heap_segment_allocated (seg);
38626 if (small_object_segments && walk_large_object_heap_p)
38629 small_object_segments = FALSE;
38630 align_const = get_alignment_constant (small_object_segments);
38631 seg = generation_start_segment (large_object_generation);
38632 x = heap_segment_mem (seg);
38633 end = heap_segment_allocated (seg);
38643 size_t s = size (x);
38644 CObjectHeader* o = (CObjectHeader*)x;
38649 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
38651 if (!fn (o->GetObjectBase(), context))
38654 x = x + Align (s, align_const);
38658 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
38660 #ifdef FEATURE_PREMORTEM_FINALIZATION
38661 finalize_queue->WalkFReachableObjects (fn);
38662 #endif //FEATURE_PREMORTEM_FINALIZATION
38665 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
38667 #ifdef MULTIPLE_HEAPS
38668 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38670 gc_heap* hp = gc_heap::g_heaps [hn];
38672 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
38675 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
38676 #endif //MULTIPLE_HEAPS
38679 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
38681 uint8_t* o = (uint8_t*)obj;
38684 go_through_object_cl (method_table (o), o, size(o), oo,
38688 Object *oh = (Object*)*oo;
38689 if (!fn (oh, context))
38697 void GCHeap::DiagWalkObject2 (Object* obj, walk_fn2 fn, void* context)
38699 uint8_t* o = (uint8_t*)obj;
38702 go_through_object_cl (method_table (o), o, size(o), oo,
38706 if (!fn (obj, oo, context))
38714 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type, int gen_number)
38716 gc_heap* hp = (gc_heap*)gc_context;
38718 if (type == walk_for_uoh)
38720 hp->walk_survivors_for_uoh (diag_context, fn, gen_number);
38724 hp->walk_survivors (fn, diag_context, type);
38728 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
38730 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
38733 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
38735 gc_heap* hp = (gc_heap*)gc_context;
38736 hp->walk_finalize_queue (fn);
38739 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
38741 #ifdef MULTIPLE_HEAPS
38742 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
38744 gc_heap* hp = gc_heap::g_heaps [hn];
38745 hp->finalize_queue->GcScanRoots(fn, hn, sc);
38748 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
38749 #endif //MULTIPLE_HEAPS
38752 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
38754 GCScan::GcScanHandlesForProfilerAndETW (gen_number, context, fn);
38757 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
38759 GCScan::GcScanDependentHandlesForProfilerAndETW (gen_number, context, fn);
38762 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
38763 // This code is designed to catch the failure to update the write barrier
38764 // The way it works is to copy the whole heap right after every GC. The write
38765 // barrier code has been modified so that it updates the shadow as well as the
38766 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
38767 // that were updated in the real heap, but not the shadow. A mismatch indicates
38768 // an error. The offending code can be found by breaking after the correct GC,
38769 // and then placing a data breakpoint on the Heap location that was updated without
38770 // going through the write barrier.
38772 // Called at process shutdown
38773 void deleteGCShadow()
38775 if (g_GCShadow != 0)
38776 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
38781 // Called at startup and right after a GC, get a snapshot of the GC Heap
38782 void initGCShadow()
38784 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
38787 size_t len = g_gc_highest_address - g_gc_lowest_address;
38788 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
38791 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
38792 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
38794 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
38795 // If after the assert we decide to allow the program to continue
38796 // running we need to be in a state that will not trigger any
38797 // additional AVs while we fail to allocate a shadow segment, i.e.
38798 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
38803 g_GCShadowEnd += len;
38806 // save the value of g_gc_lowest_address at this time. If this value changes before
38807 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
38808 // large object segment most probably), and the whole shadow segment is inconsistent.
38809 g_shadow_lowest_address = g_gc_lowest_address;
38811 //****** Copy the whole GC heap ******
38813 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
38814 // can produce a NULL result. This is because the initialization has not completed.
38816 for (int i = max_generation; i < total_generation_count; i++)
38818 generation* gen = gc_heap::generation_of (i);
38819 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
38821 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
38824 // Copy the segment
38825 uint8_t* start = heap_segment_mem(seg);
38826 uint8_t* end = heap_segment_allocated (seg);
38827 memcpy(start + delta, start, end - start);
38828 seg = heap_segment_next_rw (seg);
38833 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
38835 // test to see if 'ptr' was only updated via the write barrier.
38836 inline void testGCShadow(Object** ptr)
38838 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
38839 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
38841 // If you get this assertion, someone updated a GC pointer in the heap without
38842 // using the write barrier. To find out who, check the value of
38843 // dd_collection_count (dynamic_data_of (0)). Also
38844 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
38845 // Then put a data breakpoint for the value of 'ptr' Then check every write
38846 // to pointer between the two GCs. The last one is not using the write barrier.
38848 // If the memory of interest does not exist at system startup,
38849 // you need to set the data breakpoint right after the memory gets committed
38850 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
38851 // in the memory window. run until the memory gets mapped. Then you can set
38854 // Note a recent change, we've identified race conditions when updating the gc shadow.
38855 // Throughout the runtime, code will update an address in the gc heap, then erect the
38856 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
38857 // from multiple threads, you can hit this assert even though all involved are using the
38858 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
38859 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
38860 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
38861 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
38862 // TODO: erroneous asserts in here.
38863 if(*shadow!=INVALIDGCVALUE)
38865 #ifdef FEATURE_BASICFREEZE
38866 // Write barriers for stores of references to frozen objects may be optimized away.
38867 if (!gc_heap::frozen_object_p(*ptr))
38868 #endif // FEATURE_BASICFREEZE
38870 _ASSERTE(!"Pointer updated without using write barrier");
38876 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
38882 void testGCShadowHelper (uint8_t* x)
38884 size_t s = size (x);
38885 if (contain_pointers (x))
38887 go_through_object_nostart (method_table(x), x, s, oo,
38888 { testGCShadow((Object**) oo); });
38892 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
38893 void checkGCWriteBarrier()
38895 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
38896 // and the GC shadow segment did not track that change!
38897 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
38899 // No shadow stack, nothing to check.
38904 generation* gen = gc_heap::generation_of (max_generation);
38905 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
38907 PREFIX_ASSUME(seg != NULL);
38911 uint8_t* x = heap_segment_mem(seg);
38912 while (x < heap_segment_allocated (seg))
38914 size_t s = size (x);
38915 testGCShadowHelper (x);
38918 seg = heap_segment_next_rw (seg);
38923 // go through non-soh object heaps
38924 int alignment = get_alignment_constant(FALSE);
38925 for (int i = uoh_start_generation; i < total_generation_count; i++)
38927 generation* gen = gc_heap::generation_of (i);
38928 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
38930 PREFIX_ASSUME(seg != NULL);
38934 uint8_t* x = heap_segment_mem(seg);
38935 while (x < heap_segment_allocated (seg))
38937 size_t s = size (x);
38938 testGCShadowHelper (x);
38939 x = x + Align (s, alignment);
38941 seg = heap_segment_next_rw (seg);
38946 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
38948 #endif // !DACCESS_COMPILE
38950 #ifdef FEATURE_BASICFREEZE
38951 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
38953 #ifdef DACCESS_COMPILE
38954 UNREFERENCED_PARAMETER(seg);
38955 UNREFERENCED_PARAMETER(pvContext);
38956 UNREFERENCED_PARAMETER(pfnMethodTable);
38957 UNREFERENCED_PARAMETER(pfnObjRef);
38959 uint8_t *o = heap_segment_mem(seg);
38961 int alignment = get_alignment_constant(TRUE);
38963 while (o < heap_segment_allocated(seg))
38965 pfnMethodTable(pvContext, o);
38967 if (contain_pointers (o))
38969 go_through_object_nostart (method_table (o), o, size(o), oo,
38972 pfnObjRef(pvContext, oo);
38977 o += Align(size(o), alignment);
38979 #endif //!DACCESS_COMPILE
38981 #endif // FEATURE_BASICFREEZE
38983 #ifndef DACCESS_COMPILE
38984 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
38986 #ifdef BACKGROUND_GC
38987 if (gc_heap::background_running_p())
38989 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
38990 if (dwRet == WAIT_OBJECT_0)
38992 else if (dwRet == WAIT_TIMEOUT)
38993 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
38995 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
38996 // as there are too many layers in between. The best we can do is to return E_FAIL;
39002 #endif // !DACCESS_COMPILE
39004 void GCHeap::TemporaryEnableConcurrentGC()
39006 #ifdef BACKGROUND_GC
39007 gc_heap::temp_disable_concurrent_p = false;
39008 #endif //BACKGROUND_GC
39011 void GCHeap::TemporaryDisableConcurrentGC()
39013 #ifdef BACKGROUND_GC
39014 gc_heap::temp_disable_concurrent_p = true;
39015 #endif //BACKGROUND_GC
39018 bool GCHeap::IsConcurrentGCEnabled()
39020 #ifdef BACKGROUND_GC
39021 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
39024 #endif //BACKGROUND_GC
39027 void PopulateDacVars(GcDacVars *gcDacVars)
39029 #ifndef DACCESS_COMPILE
39030 assert(gcDacVars != nullptr);
39032 gcDacVars->major_version_number = 1;
39033 gcDacVars->minor_version_number = 0;
39034 gcDacVars->built_with_svr = &g_built_with_svr_gc;
39035 gcDacVars->build_variant = &g_build_variant;
39036 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
39037 gcDacVars->generation_size = sizeof(generation);
39038 gcDacVars->total_generation_count = total_generation_count;
39039 gcDacVars->max_gen = &g_max_generation;
39040 #ifndef MULTIPLE_HEAPS
39041 gcDacVars->mark_array = &gc_heap::mark_array;
39042 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
39043 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
39044 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
39045 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
39046 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
39047 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
39048 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
39049 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
39050 gcDacVars->oom_info = &gc_heap::oom_info;
39051 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
39052 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
39053 #ifdef GC_CONFIG_DRIVEN
39054 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
39055 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
39056 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
39057 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
39058 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
39059 #endif // GC_CONFIG_DRIVEN
39060 #ifdef HEAP_ANALYZE
39061 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
39062 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
39063 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
39064 #endif // HEAP_ANALYZE
39066 gcDacVars->n_heaps = &gc_heap::n_heaps;
39067 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
39068 #endif // MULTIPLE_HEAPS
39070 UNREFERENCED_PARAMETER(gcDacVars);
39071 #endif // DACCESS_COMPILE