1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
24 // We just needed a simple random number generator for testing.
30 static uint64_t get_rand()
32 x = (314159269*x+278281) & 0x7FFFFFFF;
36 // obtain random number in the range 0 .. r-1
37 static uint64_t get_rand(uint64_t r) {
39 uint64_t x = (uint64_t)((get_rand() * r) >> 31);
44 uint64_t gc_rand::x = 0;
46 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
47 BOOL bgc_heap_walk_for_etw_p = FALSE;
48 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
50 #if defined(FEATURE_REDHAWK)
51 #define MAYBE_UNUSED_VAR(v) v = v
53 #define MAYBE_UNUSED_VAR(v)
54 #endif // FEATURE_REDHAWK
56 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
57 #define commit_min_th (16*OS_PAGE_SIZE)
60 #define partial_size_th 100
61 #define num_partial_refs 64
63 #define partial_size_th 100
64 #define num_partial_refs 32
67 #define demotion_plug_len_th (6*1024*1024)
70 #define MARK_STACK_INITIAL_LENGTH 1024
72 #define MARK_STACK_INITIAL_LENGTH 128
75 #define LOH_PIN_QUEUE_LENGTH 100
76 #define LOH_PIN_DECAY 10
79 // Right now we support maximum 1024 procs - meaning that we will create at most
80 // that many GC threads and GC heaps.
81 #define MAX_SUPPORTED_CPUS 1024
83 #define MAX_SUPPORTED_CPUS 64
86 uint32_t yp_spin_count_unit = 0;
87 size_t loh_size_threshold = LARGE_OBJECT_SIZE;
89 #ifdef GC_CONFIG_DRIVEN
90 int compact_ratio = 0;
91 #endif //GC_CONFIG_DRIVEN
93 // See comments in reset_memory.
94 BOOL reset_mm_p = TRUE;
96 bool g_fFinalizerRunOnShutDown = false;
99 bool g_built_with_svr_gc = true;
101 bool g_built_with_svr_gc = false;
102 #endif // FEATURE_SVR_GC
104 #if defined(BUILDENV_DEBUG)
105 uint8_t g_build_variant = 0;
106 #elif defined(BUILDENV_CHECKED)
107 uint8_t g_build_variant = 1;
109 uint8_t g_build_variant = 2;
110 #endif // defined(BUILDENV_DEBUG)
112 VOLATILE(int32_t) g_no_gc_lock = -1;
114 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
115 const char * const allocation_state_str[] = {
124 "try_free_full_seg_in_bgc",
125 "try_free_after_bgc",
128 "acquire_seg_after_cg",
129 "acquire_seg_after_bgc",
130 "check_and_wait_for_bgc",
131 "trigger_full_compact_gc",
132 "trigger_ephemeral_gc",
133 "trigger_2nd_ephemeral_gc",
137 const char * const msl_take_state_str[] = {
153 #endif //TRACE_GC && !DACCESS_COMPILE
156 // Keep this in sync with the definition of gc_reason
157 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
158 static const char* const str_gc_reasons[] =
170 "induced_compacting",
173 "lowmemory_host_blocking"
176 static const char* const str_gc_pause_modes[] =
181 "sustained_low_latency",
184 #endif // defined(DT_LOG) || defined(TRACE_GC)
187 BOOL is_induced (gc_reason reason)
189 return ((reason == reason_induced) ||
190 (reason == reason_induced_noforce) ||
191 (reason == reason_lowmemory) ||
192 (reason == reason_lowmemory_blocking) ||
193 (reason == reason_induced_compacting) ||
194 (reason == reason_lowmemory_host) ||
195 (reason == reason_lowmemory_host_blocking));
199 BOOL is_induced_blocking (gc_reason reason)
201 return ((reason == reason_induced) ||
202 (reason == reason_lowmemory_blocking) ||
203 (reason == reason_induced_compacting) ||
204 (reason == reason_lowmemory_host_blocking));
207 #ifndef DACCESS_COMPILE
210 size_t GetHighPrecisionTimeStamp()
212 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
214 return (size_t)(ts / (qpf / 1000));
219 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
220 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
221 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
222 // comparison with the global min/max.
223 GCStatistics g_GCStatistics;
224 GCStatistics g_LastGCStatistics;
226 char* GCStatistics::logFileName = NULL;
227 FILE* GCStatistics::logFile = NULL;
229 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
232 if (settings.concurrent)
234 bgc.Accumulate((uint32_t)timeInMSec*1000);
237 else if (settings.background_p)
239 fgc.Accumulate((uint32_t)timeInMSec*1000);
241 if (settings.compaction)
243 assert(settings.condemned_generation < max_generation);
244 cntFGCGen[settings.condemned_generation]++;
247 #endif // BACKGROUND_GC
249 ngc.Accumulate((uint32_t)timeInMSec*1000);
251 if (settings.compaction)
253 cntNGCGen[settings.condemned_generation]++;
256 if (is_induced (settings.reason))
257 cntReasons[(int)reason_induced]++;
258 else if (settings.stress_induced)
259 cntReasons[(int)reason_gcstress]++;
261 cntReasons[(int)settings.reason]++;
264 if (settings.concurrent || !settings.background_p)
266 #endif // BACKGROUND_GC
270 #endif // BACKGROUND_GC
273 void GCStatistics::Initialize()
275 LIMITED_METHOD_CONTRACT;
276 // for efficiency sake we're taking a dependency on the layout of a C++ object
277 // with a vtable. protect against violations of our premise:
278 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
279 "The first field of GCStatistics follows the pointer sized vtable");
281 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
282 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
283 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
286 void GCStatistics::DisplayAndUpdate()
288 LIMITED_METHOD_CONTRACT;
290 if (logFileName == NULL || logFile == NULL)
295 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
297 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
299 // NGC summary (total, timing info)
300 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
302 // FGC summary (total, timing info)
303 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
306 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
308 // NGC/FGC break out by generation & compacting vs. sweeping
309 fprintf(logFile, "NGC ");
310 for (int i = max_generation; i >= 0; --i)
311 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
312 fprintf(logFile, "\n");
314 fprintf(logFile, "FGC ");
315 for (int i = max_generation-1; i >= 0; --i)
316 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
317 fprintf(logFile, "\n");
319 // Compacting vs. Sweeping break out
320 int _cntSweep = cntNGC-cntCompactNGC;
321 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
322 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
323 _cntSweep - _cntLastSweep, _cntSweep,
324 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
326 _cntSweep = cntFGC-cntCompactFGC;
327 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
328 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
329 _cntSweep - _cntLastSweep, _cntSweep,
330 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
334 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
336 if (cntReasons[reason] != 0)
337 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
338 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
341 fprintf(logFile, "\n\n");
343 // flush the log file...
347 g_LastGCStatistics = *this;
357 size_t round_up_power2 (size_t size)
359 // Get the 0-based index of the most-significant bit in size-1.
360 // If the call failed (because size-1 is zero), size must be 1,
361 // so return 1 (because 1 rounds up to itself).
362 DWORD highest_set_bit_index;
369 &highest_set_bit_index, size - 1)) { return 1; }
371 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
372 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
373 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
374 // number of bits shifts in zeros from the right, resulting in an output of zero.
375 return static_cast<size_t>(2) << highest_set_bit_index;
379 size_t round_down_power2 (size_t size)
381 // Get the 0-based index of the most-significant bit in size.
382 // If the call failed, size must be zero so return zero.
383 DWORD highest_set_bit_index;
390 &highest_set_bit_index, size)) { return 0; }
392 // Left-shift 1 by highest_set_bit_index to get back a value containing only
393 // the most-significant set bit of size, i.e. size rounded down
394 // to the next power-of-two value.
395 return static_cast<size_t>(1) << highest_set_bit_index;
398 // Get the 0-based index of the most-significant bit in the value.
399 // Returns -1 if the input value is zero (i.e. has no set bits).
401 int index_of_highest_set_bit (size_t value)
403 // Get the 0-based index of the most-significant bit in the value.
404 // If the call failed (because value is zero), return -1.
405 DWORD highest_set_bit_index;
412 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
416 int relative_index_power2_plug (size_t power2)
418 int index = index_of_highest_set_bit (power2);
419 assert (index <= MAX_INDEX_POWER2);
421 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
425 int relative_index_power2_free_space (size_t power2)
427 int index = index_of_highest_set_bit (power2);
428 assert (index <= MAX_INDEX_POWER2);
430 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
434 uint32_t bgc_alloc_spin_count = 140;
435 uint32_t bgc_alloc_spin_count_loh = 16;
436 uint32_t bgc_alloc_spin = 2;
440 void c_write (uint32_t& place, uint32_t value)
442 Interlocked::Exchange (&place, value);
446 #ifndef DACCESS_COMPILE
447 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
448 const size_t bgc_min_per_heap = 4*1024*1024;
450 int gc_heap::gchist_index = 0;
451 gc_mechanisms_store gc_heap::gchist[max_history_count];
453 #ifndef MULTIPLE_HEAPS
454 size_t gc_heap::total_promoted_bytes = 0;
455 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
456 int gc_heap::gchist_index_per_heap = 0;
457 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
458 #endif //MULTIPLE_HEAPS
460 void gc_heap::add_to_history_per_heap()
463 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
464 current_hist->gc_index = settings.gc_index;
465 current_hist->current_bgc_state = current_bgc_state;
466 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
467 current_hist->gc_time_ms = (uint32_t)elapsed;
468 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
469 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
470 current_hist->gen0_start = generation_allocation_start (generation_of (0));
471 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
473 current_hist->bgc_lowest = background_saved_lowest_address;
474 current_hist->bgc_highest = background_saved_highest_address;
475 #endif //BACKGROUND_GC
476 current_hist->fgc_lowest = lowest_address;
477 current_hist->fgc_highest = highest_address;
478 current_hist->g_lowest = g_gc_lowest_address;
479 current_hist->g_highest = g_gc_highest_address;
481 gchist_index_per_heap++;
482 if (gchist_index_per_heap == max_history_count)
484 gchist_index_per_heap = 0;
489 void gc_heap::add_to_history()
492 gc_mechanisms_store* current_settings = &gchist[gchist_index];
493 current_settings->store (&settings);
496 if (gchist_index == max_history_count)
503 #endif //DACCESS_COMPILE
504 #endif //BACKGROUND_GC
506 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
507 BOOL gc_log_on = TRUE;
509 size_t gc_log_file_size = 0;
511 size_t gc_buffer_index = 0;
512 size_t max_gc_buffers = 0;
514 static CLRCriticalSection gc_log_lock;
516 // we keep this much in a buffer and only flush when the buffer is full
517 #define gc_log_buffer_size (1024*1024)
518 uint8_t* gc_log_buffer = 0;
519 size_t gc_log_buffer_offset = 0;
521 void log_va_msg(const char *fmt, va_list args)
525 const int BUFFERSIZE = 512;
526 static char rgchBuffer[BUFFERSIZE];
527 char * pBuffer = &rgchBuffer[0];
530 int buffer_start = 1;
531 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
532 buffer_start += pid_len;
533 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
534 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args);
537 msg_len = BUFFERSIZE - buffer_start;
540 msg_len += buffer_start;
542 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
545 memset (index_str, '-', 8);
546 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
547 gc_log_buffer[gc_log_buffer_offset] = '\n';
548 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
551 if (gc_buffer_index > max_gc_buffers)
553 fseek (gc_log, 0, SEEK_SET);
556 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
558 memset (gc_log_buffer, '*', gc_log_buffer_size);
559 gc_log_buffer_offset = 0;
562 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
563 gc_log_buffer_offset += msg_len;
568 void GCLog (const char *fmt, ... )
570 if (gc_log_on && (gc_log != NULL))
574 log_va_msg (fmt, args);
578 #endif // TRACE_GC && !DACCESS_COMPILE
580 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
582 BOOL gc_config_log_on = FALSE;
583 FILE* gc_config_log = NULL;
585 // we keep this much in a buffer and only flush when the buffer is full
586 #define gc_config_log_buffer_size (1*1024) // TEMP
587 uint8_t* gc_config_log_buffer = 0;
588 size_t gc_config_log_buffer_offset = 0;
590 // For config since we log so little we keep the whole history. Also it's only
591 // ever logged by one thread so no need to synchronize.
592 void log_va_msg_config(const char *fmt, va_list args)
594 const int BUFFERSIZE = 256;
595 static char rgchBuffer[BUFFERSIZE];
596 char * pBuffer = &rgchBuffer[0];
599 int buffer_start = 1;
600 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
601 assert (msg_len != -1);
602 msg_len += buffer_start;
604 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
606 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
607 fflush(gc_config_log);
608 gc_config_log_buffer_offset = 0;
611 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
612 gc_config_log_buffer_offset += msg_len;
615 void GCLogConfig (const char *fmt, ... )
617 if (gc_config_log_on && (gc_config_log != NULL))
620 va_start( args, fmt );
621 log_va_msg_config (fmt, args);
624 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
626 #ifdef SYNCHRONIZATION_STATS
628 // Number of GCs have we done since we last logged.
629 static unsigned int gc_count_during_log;
630 // In ms. This is how often we print out stats.
631 static const unsigned int log_interval = 5000;
632 // Time (in ms) when we start a new log interval.
633 static unsigned int log_start_tick;
634 static unsigned int gc_lock_contended;
635 static int64_t log_start_hires;
636 // Cycles accumulated in SuspendEE during log_interval.
637 static uint64_t suspend_ee_during_log;
638 // Cycles accumulated in RestartEE during log_interval.
639 static uint64_t restart_ee_during_log;
640 static uint64_t gc_during_log;
642 #endif //SYNCHRONIZATION_STATS
645 init_sync_log_stats()
647 #ifdef SYNCHRONIZATION_STATS
648 if (gc_count_during_log == 0)
650 gc_heap::init_sync_stats();
651 suspend_ee_during_log = 0;
652 restart_ee_during_log = 0;
654 gc_lock_contended = 0;
656 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
657 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
659 gc_count_during_log++;
660 #endif //SYNCHRONIZATION_STATS
664 process_sync_log_stats()
666 #ifdef SYNCHRONIZATION_STATS
668 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
670 if (log_elapsed > log_interval)
672 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
673 // Print out the cycles we spent on average in each suspend and restart.
674 printf("\n_________________________________________________________________________________\n"
675 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
676 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
680 (unsigned int)(gc_during_log / gc_count_during_log),
681 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
682 (unsigned int)(restart_ee_during_log / gc_count_during_log),
683 (double)(100.0f * gc_during_log / total));
684 gc_heap::print_sync_stats(gc_count_during_log);
686 gc_count_during_log = 0;
688 #endif //SYNCHRONIZATION_STATS
691 #ifdef MULTIPLE_HEAPS
695 gc_join_init_cpu_mapping = 0,
697 gc_join_generation_determined = 2,
698 gc_join_begin_mark_phase = 3,
699 gc_join_scan_dependent_handles = 4,
700 gc_join_rescan_dependent_handles = 5,
701 gc_join_scan_sizedref_done = 6,
702 gc_join_null_dead_short_weak = 7,
703 gc_join_scan_finalization = 8,
704 gc_join_null_dead_long_weak = 9,
705 gc_join_null_dead_syncblk = 10,
706 gc_join_decide_on_compaction = 11,
707 gc_join_rearrange_segs_compaction = 12,
708 gc_join_adjust_handle_age_compact = 13,
709 gc_join_adjust_handle_age_sweep = 14,
710 gc_join_begin_relocate_phase = 15,
711 gc_join_relocate_phase_done = 16,
712 gc_join_verify_objects_done = 17,
713 gc_join_start_bgc = 18,
714 gc_join_restart_ee = 19,
715 gc_join_concurrent_overflow = 20,
716 gc_join_suspend_ee = 21,
717 gc_join_bgc_after_ephemeral = 22,
718 gc_join_allow_fgc = 23,
719 gc_join_bgc_sweep = 24,
720 gc_join_suspend_ee_verify = 25,
721 gc_join_restart_ee_verify = 26,
722 gc_join_set_state_free = 27,
723 gc_r_join_update_card_bundle = 28,
724 gc_join_after_absorb = 29,
725 gc_join_verify_copy_table = 30,
726 gc_join_after_reset = 31,
727 gc_join_after_ephemeral_sweep = 32,
728 gc_join_after_profiler_heap_walk = 33,
729 gc_join_minimal_gc = 34,
730 gc_join_after_commit_soh_no_gc = 35,
731 gc_join_expand_loh_no_gc = 36,
732 gc_join_final_no_gc = 37,
733 gc_join_disable_software_write_watch = 38,
739 join_flavor_server_gc = 0,
743 #define first_thread_arrived 2
744 #pragma warning(push)
745 #pragma warning(disable:4324) // don't complain if DECLSPEC_ALIGN actually pads
746 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
748 // Shared non volatile keep on separate line to prevent eviction
751 // Keep polling/wait structures on separate line write once per join
752 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
753 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
754 Volatile<int> lock_color;
755 VOLATILE(BOOL) wait_done;
756 VOLATILE(BOOL) joined_p;
758 // Keep volatile counted locks on separate cache line write many per join
759 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
760 VOLATILE(int32_t) join_lock;
761 VOLATILE(int32_t) r_join_lock;
771 type_first_r_join = 3,
783 join_heap_restart = 100,
784 join_heap_r_restart = 200
796 join_structure join_struct;
799 gc_join_flavor flavor;
802 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
803 // remember join id and last thread to arrive so restart can use these
805 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
807 // counters for joins, in 1000's of clock cycles
808 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
812 BOOL init (int n_th, gc_join_flavor f)
814 dprintf (JOIN_LOG, ("Initializing join structure"));
815 join_struct.n_threads = n_th;
816 join_struct.lock_color = 0;
817 for (int i = 0; i < 3; i++)
819 if (!join_struct.joined_event[i].IsValid())
821 join_struct.joined_p = FALSE;
822 dprintf (JOIN_LOG, ("Creating join event %d", i));
823 // TODO - changing this to a non OS event
824 // because this is also used by BGC threads which are
825 // managed threads and WaitEx does not allow you to wait
826 // for an OS event on a managed thread.
827 // But we are not sure if this plays well in the hosting
829 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
830 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
834 join_struct.join_lock = join_struct.n_threads;
835 join_struct.r_join_lock = join_struct.n_threads;
836 join_struct.wait_done = FALSE;
840 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
848 dprintf (JOIN_LOG, ("Destroying join structure"));
849 for (int i = 0; i < 3; i++)
851 if (join_struct.joined_event[i].IsValid())
852 join_struct.joined_event[i].CloseEvent();
856 inline void fire_event (int heap, join_time time, join_type type, int join_id)
858 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
861 void join (gc_heap* gch, int join_id)
864 // parallel execution ends here
865 end[gch->heap_number] = get_ts();
868 assert (!join_struct.joined_p);
869 int color = join_struct.lock_color.LoadWithoutBarrier();
871 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
873 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
874 flavor, join_id, (int32_t)(join_struct.join_lock)));
876 fire_event (gch->heap_number, time_start, type_join, join_id);
878 //busy wait around the color
879 if (color == join_struct.lock_color.LoadWithoutBarrier())
882 int spin_count = 128 * yp_spin_count_unit;
883 for (int j = 0; j < spin_count; j++)
885 if (color != join_struct.lock_color.LoadWithoutBarrier())
889 YieldProcessor(); // indicate to the processor that we are spinning
892 // we've spun, and if color still hasn't changed, fall into hard wait
893 if (color == join_struct.lock_color.LoadWithoutBarrier())
895 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
896 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
898 //Thread* current_thread = GCToEEInterface::GetThread();
899 //BOOL cooperative_mode = gc_heap::enable_preemptive ();
900 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
901 //gc_heap::disable_preemptive (cooperative_mode);
903 if (dwJoinWait != WAIT_OBJECT_0)
905 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
910 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
911 if (color == join_struct.lock_color.LoadWithoutBarrier())
916 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
917 flavor, join_id, (int32_t)(join_struct.join_lock)));
920 fire_event (gch->heap_number, time_end, type_join, join_id);
923 // parallel execution starts here
924 start[gch->heap_number] = get_ts();
925 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
930 fire_event (gch->heap_number, time_start, type_last_join, join_id);
932 join_struct.joined_p = TRUE;
933 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
934 join_struct.joined_event[!color].Reset();
936 // this one is alone so it can proceed
938 // remember the join id, the last thread arriving, the start of the sequential phase,
939 // and keep track of the cycles spent waiting in the join
940 thd = gch->heap_number;
941 start_seq = get_ts();
942 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
947 // Reverse join - first thread gets here does the work; other threads will only proceed
948 // after the work is done.
949 // Note that you cannot call this twice in a row on the same thread. Plus there's no
950 // need to call it twice in row - you should just merge the work.
951 BOOL r_join (gc_heap* gch, int join_id)
954 if (join_struct.n_threads == 1)
959 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
961 if (!join_struct.wait_done)
963 dprintf (JOIN_LOG, ("r_join() Waiting..."));
965 fire_event (gch->heap_number, time_start, type_join, join_id);
967 //busy wait around the color
968 if (!join_struct.wait_done)
971 int spin_count = 256 * yp_spin_count_unit;
972 for (int j = 0; j < spin_count; j++)
974 if (join_struct.wait_done)
978 YieldProcessor(); // indicate to the processor that we are spinning
981 // we've spun, and if color still hasn't changed, fall into hard wait
982 if (!join_struct.wait_done)
984 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
985 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
986 if (dwJoinWait != WAIT_OBJECT_0)
988 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
993 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
994 if (!join_struct.wait_done)
999 dprintf (JOIN_LOG, ("r_join() done"));
1002 fire_event (gch->heap_number, time_end, type_join, join_id);
1009 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
1017 return GCToOSInterface::QueryPerformanceCounter();
1020 void start_ts (gc_heap* gch)
1022 // parallel execution ends here
1023 start[gch->heap_number] = get_ts();
1030 uint64_t elapsed_seq = get_ts() - start_seq;
1031 uint64_t max = 0, sum = 0, wake = 0;
1032 uint64_t min_ts = start[0];
1033 for (int i = 1; i < join_struct.n_threads; i++)
1035 if(min_ts > start[i]) min_ts = start[i];
1038 for (int i = 0; i < join_struct.n_threads; i++)
1040 uint64_t wake_delay = start[i] - min_ts;
1041 uint64_t elapsed = end[i] - start[i];
1047 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1048 uint64_t par_loss = join_struct.n_threads*max - sum;
1049 double efficiency = 0.0;
1051 efficiency = sum*100.0/(join_struct.n_threads*max);
1053 const double ts_scale = 1e-6;
1055 // enable this printf to get statistics on each individual join as it occurs
1056 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1058 elapsed_total[id] += sum;
1059 wake_total[id] += wake;
1060 seq_loss_total[id] += seq_loss;
1061 par_loss_total[id] += par_loss;
1063 // every 10 seconds, print a summary of the time spent in each type of join
1064 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1066 printf("**** summary *****\n");
1067 for (int i = 0; i < 16; i++)
1069 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1071 ts_scale*elapsed_total[i],
1072 ts_scale*wake_total[i],
1073 ts_scale*seq_loss_total[i],
1074 ts_scale*par_loss_total[i],
1075 ts_scale*in_join_total[i]);
1076 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1078 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1082 fire_event (join_heap_restart, time_start, type_restart, -1);
1083 assert (join_struct.joined_p);
1084 join_struct.joined_p = FALSE;
1085 join_struct.join_lock = join_struct.n_threads;
1086 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1087 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1088 int color = join_struct.lock_color.LoadWithoutBarrier();
1089 join_struct.lock_color = !color;
1090 join_struct.joined_event[color].Set();
1092 // printf("Set joined_event %d\n", !join_struct.lock_color);
1094 fire_event (join_heap_restart, time_end, type_restart, -1);
1097 start[thd] = get_ts();
1103 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1104 return join_struct.joined_p;
1109 if (join_struct.n_threads != 1)
1111 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1112 join_struct.wait_done = TRUE;
1113 join_struct.joined_event[first_thread_arrived].Set();
1114 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1120 if (join_struct.n_threads != 1)
1122 join_struct.r_join_lock = join_struct.n_threads;
1123 join_struct.wait_done = FALSE;
1124 join_struct.joined_event[first_thread_arrived].Reset();
1131 #ifdef BACKGROUND_GC
1133 #endif //BACKGROUND_GC
1135 #endif //MULTIPLE_HEAPS
1137 #define spin_and_switch(count_to_spin, expr) \
1139 for (int j = 0; j < count_to_spin; j++) \
1149 GCToOSInterface::YieldThread(0); \
1153 #ifndef DACCESS_COMPILE
1154 #ifdef BACKGROUND_GC
1156 #define max_pending_allocs 64
1158 class exclusive_sync
1160 // TODO - verify that this is the right syntax for Volatile.
1161 VOLATILE(uint8_t*) rwp_object;
1162 VOLATILE(int32_t) needs_checking;
1166 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1168 // TODO - perhaps each object should be on its own cache line...
1169 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1171 int find_free_index ()
1173 for (int i = 0; i < max_pending_allocs; i++)
1175 if (alloc_objects [i] == (uint8_t*)0)
1187 spin_count = 32 * (g_num_processors - 1);
1190 for (int i = 0; i < max_pending_allocs; i++)
1192 alloc_objects [i] = (uint8_t*)0;
1198 for (int i = 0; i < max_pending_allocs; i++)
1200 if (alloc_objects [i] != (uint8_t*)0)
1202 GCToOSInterface::DebugBreak();
1207 void bgc_mark_set (uint8_t* obj)
1209 dprintf (3, ("cm: probing %Ix", obj));
1211 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1213 // If we spend too much time spending all the allocs,
1214 // consider adding a high water mark and scan up
1215 // to that; we'll need to interlock in done when
1216 // we update the high watermark.
1217 for (int i = 0; i < max_pending_allocs; i++)
1219 if (obj == alloc_objects[i])
1222 dprintf (3, ("cm: will spin"));
1223 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1230 dprintf (3, ("cm: set %Ix", obj));
1235 spin_and_switch (spin_count, (needs_checking == 0));
1240 int loh_alloc_set (uint8_t* obj)
1242 if (!gc_heap::cm_in_progress)
1248 dprintf (3, ("loh alloc: probing %Ix", obj));
1250 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1252 if (obj == rwp_object)
1255 spin_and_switch (spin_count, (obj != rwp_object));
1260 int cookie = find_free_index();
1264 alloc_objects[cookie] = obj;
1268 // GCToOSInterface::DebugBreak();
1271 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1277 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1278 spin_and_switch (spin_count, (find_free_index () != -1));
1285 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1286 spin_and_switch (spin_count, (needs_checking == 0));
1291 void bgc_mark_done ()
1293 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1297 void loh_alloc_done_with_index (int index)
1299 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1300 assert ((index >= 0) && (index < max_pending_allocs));
1301 alloc_objects[index] = (uint8_t*)0;
1304 void loh_alloc_done (uint8_t* obj)
1306 #ifdef BACKGROUND_GC
1307 if (!gc_heap::cm_in_progress)
1312 for (int i = 0; i < max_pending_allocs; i++)
1314 if (alloc_objects [i] == obj)
1316 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1317 alloc_objects[i] = (uint8_t*)0;
1321 #endif //BACKGROUND_GC
1325 // Note that this class was written assuming just synchronization between
1326 // one background GC thread and multiple user threads that might request
1327 // an FGC - it does not take into account what kind of locks the multiple
1328 // user threads might be holding at the time (eg, there could only be one
1329 // user thread requesting an FGC because it needs to take gc_lock first)
1330 // so you'll see checks that may not be necessary if you take those conditions
1331 // into consideration.
1333 // With the introduction of Server Background GC we no longer use this
1334 // class to do synchronization between FGCs and BGC.
1335 class recursive_gc_sync
1337 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1338 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1339 static VOLATILE(int32_t) foreground_count; // initial state 0;
1340 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1341 static GCEvent foreground_complete;//Auto Reset
1342 static GCEvent foreground_allowed;//Auto Reset
1344 static void begin_background();
1345 static void end_background();
1346 static void begin_foreground();
1347 static void end_foreground();
1348 BOOL allow_foreground ();
1350 static void shutdown();
1351 static BOOL background_running_p() {return gc_background_running;}
1354 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1355 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1356 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1357 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1358 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1359 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1361 BOOL recursive_gc_sync::init ()
1363 foreground_request_count = 0;
1364 foreground_count = 0;
1365 gc_background_running = FALSE;
1366 foreground_gate = 0;
1368 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1372 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1384 void recursive_gc_sync::shutdown()
1386 if (foreground_complete.IsValid())
1387 foreground_complete.CloseEvent();
1388 if (foreground_allowed.IsValid())
1389 foreground_allowed.CloseEvent();
1392 void recursive_gc_sync::begin_background()
1394 dprintf (2, ("begin background"));
1395 foreground_request_count = 1;
1396 foreground_count = 1;
1397 foreground_allowed.Reset();
1398 gc_background_running = TRUE;
1400 void recursive_gc_sync::end_background()
1402 dprintf (2, ("end background"));
1403 gc_background_running = FALSE;
1404 foreground_gate = 1;
1405 foreground_allowed.Set();
1408 void recursive_gc_sync::begin_foreground()
1410 dprintf (2, ("begin_foreground"));
1412 bool cooperative_mode = false;
1413 if (gc_background_running)
1415 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1416 gc_heap::alloc_wait_event_p = TRUE;
1420 Interlocked::Increment (&foreground_request_count);
1423 dprintf(2, ("Waiting sync gc point"));
1424 assert (foreground_allowed.IsValid());
1425 assert (foreground_complete.IsValid());
1427 cooperative_mode = gc_heap::enable_preemptive ();
1429 foreground_allowed.Wait(INFINITE, FALSE);
1431 dprintf(2, ("Waiting sync gc point is done"));
1433 gc_heap::disable_preemptive (cooperative_mode);
1435 if (foreground_gate)
1437 Interlocked::Increment (&foreground_count);
1438 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1439 if (foreground_gate)
1441 gc_heap::settings.concurrent = FALSE;
1452 goto try_again_no_inc;
1457 void recursive_gc_sync::end_foreground()
1459 dprintf (2, ("end_foreground"));
1460 if (gc_background_running)
1462 Interlocked::Decrement (&foreground_request_count);
1463 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1464 if (Interlocked::Decrement (&foreground_count) == 0)
1466 //c_write ((BOOL*)&foreground_gate, 0);
1467 // TODO - couldn't make the syntax work with Volatile<T>
1468 foreground_gate = 0;
1469 if (foreground_count == 0)
1471 foreground_allowed.Reset ();
1472 dprintf(2, ("setting foreground complete event"));
1473 foreground_complete.Set();
1480 BOOL recursive_gc_sync::allow_foreground()
1482 assert (gc_heap::settings.concurrent);
1483 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1484 (int32_t)foreground_request_count, (int32_t)foreground_count));
1486 BOOL did_fgc = FALSE;
1488 //if we have suspended the EE, just return because
1489 //some thread could be waiting on this to proceed.
1490 if (!GCHeap::GcInProgress)
1492 //TODO BACKGROUND_GC This is to stress the concurrency between
1493 //background and foreground
1494 // gc_heap::disallow_new_allocation (0);
1496 //GCToOSInterface::YieldThread(0);
1499 if (foreground_request_count != 0)
1501 //foreground wants to run
1502 //save the important settings
1503 //TODO BACKGROUND_GC be more selective about the important settings.
1504 gc_mechanisms saved_settings = gc_heap::settings;
1508 //c_write ((BOOL*)&foreground_gate, 1);
1509 // TODO - couldn't make the syntax work with Volatile<T>
1510 foreground_gate = 1;
1511 foreground_allowed.Set ();
1512 foreground_complete.Wait (INFINITE, FALSE);
1513 }while (/*foreground_request_count ||*/ foreground_gate);
1515 assert (!foreground_gate);
1517 //restore the important settings
1518 gc_heap::settings = saved_settings;
1519 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1520 //the background GC shouldn't be using gc_high and gc_low
1521 //gc_low = lowest_address;
1522 //gc_high = highest_address;
1525 //TODO BACKGROUND_GC This is to stress the concurrency between
1526 //background and foreground
1527 // gc_heap::allow_new_allocation (0);
1531 dprintf (100, ("leave allow_foreground"));
1532 assert (gc_heap::settings.concurrent);
1536 #endif //BACKGROUND_GC
1537 #endif //DACCESS_COMPILE
1540 #if defined(COUNT_CYCLES)
1542 #pragma warning(disable:4035)
1546 unsigned GetCycleCount32() // enough for about 40 seconds
1554 #pragma warning(default:4035)
1556 #endif //COUNT_CYCLES
1559 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1562 #ifndef MULTIPLE_HEAPS
1564 #endif // MULTIPLE_HEAPS
1566 void reset_memory (uint8_t* o, size_t sizeo);
1570 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1571 static bool virtual_alloc_hardware_write_watch = false;
1572 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1574 static bool hardware_write_watch_capability = false;
1576 #ifndef DACCESS_COMPILE
1578 //check if the write watch APIs are supported.
1580 void hardware_write_watch_api_supported()
1582 if (GCToOSInterface::SupportsWriteWatch())
1584 hardware_write_watch_capability = true;
1585 dprintf (2, ("WriteWatch supported"));
1589 dprintf (2,("WriteWatch not supported"));
1593 #endif //!DACCESS_COMPILE
1595 inline bool can_use_hardware_write_watch()
1597 return hardware_write_watch_capability;
1600 inline bool can_use_write_watch_for_gc_heap()
1602 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1604 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1605 return can_use_hardware_write_watch();
1606 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1609 inline bool can_use_write_watch_for_card_table()
1611 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1614 return can_use_hardware_write_watch();
1619 #define mem_reserve (MEM_RESERVE)
1620 #endif //WRITE_WATCH
1622 //check if the low memory notification is supported
1624 #ifndef DACCESS_COMPILE
1626 void WaitLongerNoInstru (int i)
1628 // every 8th attempt:
1629 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1631 // if we're waiting for gc to finish, we should block immediately
1632 if (g_fSuspensionPending == 0)
1634 if (g_num_processors > 1)
1636 YieldProcessor(); // indicate to the processor that we are spinning
1638 GCToOSInterface::YieldThread (0);
1640 GCToOSInterface::Sleep (5);
1643 GCToOSInterface::Sleep (5);
1646 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1647 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1648 // It is important that the thread is going to wait for GC. Otherwise the thread
1649 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1653 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1654 // started, it is important to block until the GC thread calls set_gc_done (since it is
1655 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1656 // conditions which can otherwise occur if threads are allowed to spin in this function
1657 // (and therefore starve the GC thread) between the point when the GC thread sets the
1658 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1659 if (gc_heap::gc_started)
1661 gc_heap::wait_for_gc_done();
1664 GCToEEInterface::DisablePreemptiveGC();
1666 else if (g_fSuspensionPending > 0)
1668 g_theGCHeap->WaitUntilGCComplete();
1673 static void safe_switch_to_thread()
1675 bool cooperative_mode = gc_heap::enable_preemptive();
1677 GCToOSInterface::YieldThread(0);
1679 gc_heap::disable_preemptive(cooperative_mode);
1683 // We need the following methods to have volatile arguments, so that they can accept
1684 // raw pointers in addition to the results of the & operator on Volatile<T>.
1687 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1691 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1694 while (VolatileLoad(lock) >= 0)
1696 if ((++i & 7) && !IsGCInProgress())
1698 if (g_num_processors > 1)
1700 #ifndef MULTIPLE_HEAPS
1701 int spin_count = 32 * yp_spin_count_unit;
1702 #else //!MULTIPLE_HEAPS
1703 int spin_count = yp_spin_count_unit;
1704 #endif //!MULTIPLE_HEAPS
1705 for (int j = 0; j < spin_count; j++)
1707 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1709 YieldProcessor(); // indicate to the processor that we are spinning
1711 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1713 safe_switch_to_thread();
1718 safe_switch_to_thread();
1723 WaitLongerNoInstru(i);
1731 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1733 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1737 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1739 VolatileStore<int32_t>((int32_t*)lock, -1);
1745 static void enter_spin_lock(GCSpinLock *pSpinLock)
1747 enter_spin_lock_noinstru(&pSpinLock->lock);
1748 assert (pSpinLock->holding_thread == (Thread*)-1);
1749 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1753 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1755 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1757 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1762 static void leave_spin_lock(GCSpinLock *pSpinLock)
1764 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1765 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1766 pSpinLock->released_by_gc_p = gc_thread_p;
1767 pSpinLock->holding_thread = (Thread*) -1;
1768 if (pSpinLock->lock != -1)
1769 leave_spin_lock_noinstru(&pSpinLock->lock);
1772 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1773 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1775 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1776 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1780 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1781 //the gc thread call WaitLonger.
1782 void WaitLonger (int i
1783 #ifdef SYNCHRONIZATION_STATS
1784 , GCSpinLock* spin_lock
1785 #endif //SYNCHRONIZATION_STATS
1788 #ifdef SYNCHRONIZATION_STATS
1789 (spin_lock->num_wait_longer)++;
1790 #endif //SYNCHRONIZATION_STATS
1792 // every 8th attempt:
1793 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1796 // if we're waiting for gc to finish, we should block immediately
1797 if (!gc_heap::gc_started)
1799 #ifdef SYNCHRONIZATION_STATS
1800 (spin_lock->num_switch_thread_w)++;
1801 #endif //SYNCHRONIZATION_STATS
1802 if (g_num_processors > 1)
1804 YieldProcessor(); // indicate to the processor that we are spinning
1806 GCToOSInterface::YieldThread (0);
1808 GCToOSInterface::Sleep (5);
1811 GCToOSInterface::Sleep (5);
1814 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1815 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1816 // It is important that the thread is going to wait for GC. Otherwise the thread
1817 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1818 if (gc_heap::gc_started)
1820 gc_heap::wait_for_gc_done();
1825 #ifdef SYNCHRONIZATION_STATS
1826 (spin_lock->num_disable_preemptive_w)++;
1827 #endif //SYNCHRONIZATION_STATS
1828 GCToEEInterface::DisablePreemptiveGC();
1833 static void enter_spin_lock (GCSpinLock* spin_lock)
1837 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1840 while (spin_lock->lock >= 0)
1842 if ((++i & 7) && !gc_heap::gc_started)
1844 if (g_num_processors > 1)
1846 #ifndef MULTIPLE_HEAPS
1847 int spin_count = 32 * yp_spin_count_unit;
1848 #else //!MULTIPLE_HEAPS
1849 int spin_count = yp_spin_count_unit;
1850 #endif //!MULTIPLE_HEAPS
1851 for (int j = 0; j < spin_count; j++)
1853 if (spin_lock->lock < 0 || gc_heap::gc_started)
1855 YieldProcessor(); // indicate to the processor that we are spinning
1857 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1859 #ifdef SYNCHRONIZATION_STATS
1860 (spin_lock->num_switch_thread)++;
1861 #endif //SYNCHRONIZATION_STATS
1862 bool cooperative_mode = gc_heap::enable_preemptive ();
1864 GCToOSInterface::YieldThread(0);
1866 gc_heap::disable_preemptive (cooperative_mode);
1870 GCToOSInterface::YieldThread(0);
1875 #ifdef SYNCHRONIZATION_STATS
1877 #endif //SYNCHRONIZATION_STATS
1885 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1887 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1891 static void leave_spin_lock (GCSpinLock * spin_lock)
1893 spin_lock->lock = -1;
1896 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1900 bool gc_heap::enable_preemptive ()
1902 return GCToEEInterface::EnablePreemptiveGC();
1905 void gc_heap::disable_preemptive (bool restore_cooperative)
1907 if (restore_cooperative)
1909 GCToEEInterface::DisablePreemptiveGC();
1913 #endif // !DACCESS_COMPILE
1915 typedef void ** PTR_PTR;
1916 //This function clears a piece of memory
1917 // size has to be Dword aligned
1920 void memclr ( uint8_t* mem, size_t size)
1922 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1923 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1924 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1927 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1928 // memset directly to make it obvious what's going on.
1929 PTR_PTR m = (PTR_PTR) mem;
1930 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1934 memset (mem, 0, size);
1937 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1939 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1940 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1941 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1943 // size must be a multiple of the pointer size
1944 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1945 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1947 // copy in groups of four pointer sized things at a time
1952 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1953 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1954 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1955 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1959 while ((size -= sz4ptr) >= sz4ptr);
1962 // still two pointer sized things or more left to copy?
1965 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1966 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1971 // still one pointer sized thing left to copy?
1974 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1982 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1984 return ((add / pitch) * pitch);
1987 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1988 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1989 // i.e, if a larger alignment matters or is beneficial, the compiler
1990 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1991 // converse - it's a heuristic for the GC to use a larger alignment.
1992 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1995 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1996 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1999 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
2000 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
2004 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
2006 #ifdef RESPECT_LARGE_ALIGNMENT
2007 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
2009 UNREFERENCED_PARAMETER(p1);
2010 UNREFERENCED_PARAMETER(p2);
2012 #endif //RESPECT_LARGE_ALIGNMENT
2016 size_t switch_alignment_size (BOOL already_padded_p)
2018 if (already_padded_p)
2019 return DATA_ALIGNMENT;
2021 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
2025 #ifdef FEATURE_STRUCTALIGN
2026 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
2027 void clear_node_aligninfo (uint8_t *node);
2028 #else // FEATURE_STRUCTALIGN
2029 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
2030 void set_node_realigned (uint8_t* node);
2031 void clear_node_realigned(uint8_t* node);
2032 #endif // FEATURE_STRUCTALIGN
2035 size_t AlignQword (size_t nbytes)
2037 #ifdef FEATURE_STRUCTALIGN
2038 // This function is used to align everything on the large object
2039 // heap to an 8-byte boundary, to reduce the number of unaligned
2040 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
2041 // the compiler dictates the optimal alignment instead of having
2042 // a heuristic in the GC.
2043 return Align (nbytes);
2044 #else // FEATURE_STRUCTALIGN
2045 return (nbytes + 7) & ~7;
2046 #endif // FEATURE_STRUCTALIGN
2050 BOOL Aligned (size_t n)
2052 return (n & ALIGNCONST) == 0;
2055 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2057 #ifdef FEATURE_STRUCTALIGN
2058 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2059 #else // FEATURE_STRUCTALIGN
2060 #define MAX_STRUCTALIGN 0
2061 #endif // FEATURE_STRUCTALIGN
2063 #ifdef FEATURE_STRUCTALIGN
2065 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2067 // The resulting alignpad must be either 0 or at least min_obj_size.
2068 // Note that by computing the following difference on unsigned types,
2069 // we can do the range check 0 < alignpad < min_obj_size with a
2070 // single conditional branch.
2071 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2073 return requiredAlignment;
2079 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2081 // required alignment must be a power of two
2082 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2083 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2084 _ASSERTE(requiredAlignment >= sizeof(void *));
2085 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2087 // When this method is invoked for individual objects (i.e., alignmentOffset
2088 // is just the size of the PostHeader), what needs to be aligned when
2089 // we're done is the pointer to the payload of the object (which means
2090 // the actual resulting object pointer is typically not aligned).
2092 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2093 ptrdiff_t alignpad = result - origPtr;
2095 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2099 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2101 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2104 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2106 return StructAlign (ptr, requiredAlignment) == ptr;
2110 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2112 if (requiredAlignment == DATA_ALIGNMENT)
2114 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2115 // alignment padding object), the worst-case alignment padding is correspondingly larger
2116 // than the required alignment.
2117 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2121 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2123 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2125 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2126 // for padding before the actual object, it also leaves space for filling a gap after the
2127 // actual object. This is needed on the large object heap, as the outer allocation functions
2128 // don't operate on an allocation context (which would have left space for the final gap).
2129 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2132 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2134 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2135 if (alignedPtr != newAlloc) {
2136 make_unused_array (newAlloc, alignedPtr - newAlloc);
2138 acontext->alloc_ptr = alignedPtr + Align (size);
2142 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2144 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2145 if (alignedPtr != newAlloc) {
2146 make_unused_array (newAlloc, alignedPtr - newAlloc);
2148 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2149 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2153 #else // FEATURE_STRUCTALIGN
2154 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2155 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2156 #endif // FEATURE_STRUCTALIGN
2158 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2160 #define CLR_SIZE ((size_t)(8*1024))
2162 #define CLR_SIZE ((size_t)(8*1024))
2165 #define END_SPACE_AFTER_GC (loh_size_threshold + MAX_STRUCTALIGN)
2167 #ifdef BACKGROUND_GC
2168 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2170 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2171 #endif //BACKGROUND_GC
2173 // This is always power of 2.
2174 const size_t min_segment_size_hard_limit = 1024*1024*16;
2180 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2181 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2185 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2186 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2194 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2195 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2199 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2200 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2206 //amount in bytes of the etw allocation tick
2207 const size_t etw_allocation_tick = 100*1024;
2209 const size_t low_latency_alloc = 256*1024;
2211 const size_t fgn_check_quantum = 2*1024*1024;
2214 const int max_snoop_level = 128;
2219 //threshold of heap size to turn on card bundles.
2220 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2221 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2222 #endif //CARD_BUNDLE
2224 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2227 size_t align_on_page (size_t add)
2229 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2233 uint8_t* align_on_page (uint8_t* add)
2235 return (uint8_t*)align_on_page ((size_t) add);
2239 size_t align_lower_page (size_t add)
2241 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2245 uint8_t* align_lower_page (uint8_t* add)
2247 return (uint8_t*)align_lower_page ((size_t)add);
2251 size_t align_write_watch_lower_page (size_t add)
2253 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2257 uint8_t* align_write_watch_lower_page (uint8_t* add)
2259 return (uint8_t*)align_lower_page ((size_t)add);
2264 BOOL power_of_two_p (size_t integer)
2266 return !(integer & (integer-1));
2270 BOOL oddp (size_t integer)
2272 return (integer & 1) != 0;
2275 // we only ever use this for WORDs.
2276 size_t logcount (size_t word)
2278 //counts the number of high bits in a 16 bit word.
2279 assert (word < 0x10000);
2281 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2282 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2283 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2284 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2288 #ifndef DACCESS_COMPILE
2290 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2292 WriteBarrierParameters args = {};
2293 args.operation = WriteBarrierOp::StompResize;
2294 args.is_runtime_suspended = is_runtime_suspended;
2295 args.requires_upper_bounds_check = requires_upper_bounds_check;
2297 args.card_table = g_gc_card_table;
2298 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2299 args.card_bundle_table = g_gc_card_bundle_table;
2302 args.lowest_address = g_gc_lowest_address;
2303 args.highest_address = g_gc_highest_address;
2305 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2306 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2308 args.write_watch_table = g_gc_sw_ww_table;
2310 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2312 GCToEEInterface::StompWriteBarrier(&args);
2315 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2317 WriteBarrierParameters args = {};
2318 args.operation = WriteBarrierOp::StompEphemeral;
2319 args.is_runtime_suspended = true;
2320 args.ephemeral_low = ephemeral_low;
2321 args.ephemeral_high = ephemeral_high;
2322 GCToEEInterface::StompWriteBarrier(&args);
2325 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2327 WriteBarrierParameters args = {};
2328 args.operation = WriteBarrierOp::Initialize;
2329 args.is_runtime_suspended = true;
2330 args.requires_upper_bounds_check = false;
2331 args.card_table = g_gc_card_table;
2333 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2334 args.card_bundle_table = g_gc_card_bundle_table;
2337 args.lowest_address = g_gc_lowest_address;
2338 args.highest_address = g_gc_highest_address;
2339 args.ephemeral_low = ephemeral_low;
2340 args.ephemeral_high = ephemeral_high;
2341 GCToEEInterface::StompWriteBarrier(&args);
2344 #endif // DACCESS_COMPILE
2346 //extract the low bits [0,low[ of a uint32_t
2347 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2348 //extract the high bits [high, 32] of a uint32_t
2349 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2351 // Things we need to manually initialize:
2352 // gen0 min_size - based on cache
2353 // gen0/1 max_size - based on segment size
2354 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2356 // latency_level_memory_footprint
2359 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2361 {160*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2363 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2365 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2368 // latency_level_balanced
2372 #ifdef MULTIPLE_HEAPS
2376 #endif //MULTIPLE_HEAPS
2379 {256*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2381 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2383 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2390 class CObjectHeader;
2394 class c_synchronize;
2396 #ifdef FEATURE_PREMORTEM_FINALIZATION
2397 #ifndef DACCESS_COMPILE
2399 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2400 #endif //!DACCESS_COMPILE
2401 #endif // FEATURE_PREMORTEM_FINALIZATION
2403 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2406 #ifdef USE_INTROSORT
2407 #define _sort introsort::sort
2408 #else //USE_INTROSORT
2409 #define _sort qsort1
2410 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2411 #endif //USE_INTROSORT
2413 void* virtual_alloc (size_t size);
2414 void virtual_free (void* add, size_t size);
2416 /* per heap static initialization */
2418 #ifndef MULTIPLE_HEAPS
2419 uint32_t* gc_heap::mark_array;
2420 #endif //MULTIPLE_HEAPS
2424 uint8_t** gc_heap::g_mark_list;
2426 #ifdef PARALLEL_MARK_LIST_SORT
2427 uint8_t** gc_heap::g_mark_list_copy;
2428 #endif //PARALLEL_MARK_LIST_SORT
2430 size_t gc_heap::mark_list_size;
2433 #ifdef SEG_MAPPING_TABLE
2434 seg_mapping* seg_mapping_table;
2435 #endif //SEG_MAPPING_TABLE
2437 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2438 sorted_table* gc_heap::seg_table;
2439 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2441 #ifdef MULTIPLE_HEAPS
2442 GCEvent gc_heap::ee_suspend_event;
2443 size_t gc_heap::min_balance_threshold = 0;
2444 #endif //MULTIPLE_HEAPS
2446 VOLATILE(BOOL) gc_heap::gc_started;
2448 #ifdef MULTIPLE_HEAPS
2450 GCEvent gc_heap::gc_start_event;
2451 bool gc_heap::gc_thread_no_affinitize_p = false;
2452 uintptr_t process_mask = 0;
2454 int gc_heap::n_heaps;
2456 gc_heap** gc_heap::g_heaps;
2458 size_t* gc_heap::g_promoted;
2461 int* gc_heap::g_mark_stack_busy;
2465 #ifdef BACKGROUND_GC
2466 size_t* gc_heap::g_bpromoted;
2467 #endif //BACKGROUND_GC
2469 #else //MULTIPLE_HEAPS
2471 size_t gc_heap::g_promoted;
2473 #ifdef BACKGROUND_GC
2474 size_t gc_heap::g_bpromoted;
2475 #endif //BACKGROUND_GC
2477 #endif //MULTIPLE_HEAPS
2479 size_t gc_heap::reserved_memory = 0;
2480 size_t gc_heap::reserved_memory_limit = 0;
2481 BOOL gc_heap::g_low_memory_status;
2483 #ifndef DACCESS_COMPILE
2484 static gc_reason gc_trigger_reason = reason_empty;
2485 #endif //DACCESS_COMPILE
2487 gc_latency_level gc_heap::latency_level = latency_level_default;
2489 gc_mechanisms gc_heap::settings;
2491 gc_history_global gc_heap::gc_data_global;
2493 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2495 size_t gc_heap::gc_gen0_desired_high;
2497 CLRCriticalSection gc_heap::check_commit_cs;
2499 size_t gc_heap::current_total_committed = 0;
2501 size_t gc_heap::current_total_committed_bookkeeping = 0;
2504 double gc_heap::short_plugs_pad_ratio = 0;
2505 #endif //SHORT_PLUGS
2508 #define MAX_ALLOWED_MEM_LOAD 85
2510 // consider putting this in dynamic data -
2511 // we may want different values for workstation
2513 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2515 size_t gc_heap::youngest_gen_desired_th;
2518 uint32_t gc_heap::last_gc_memory_load = 0;
2520 size_t gc_heap::last_gc_heap_size = 0;
2522 size_t gc_heap::last_gc_fragmentation = 0;
2524 uint64_t gc_heap::mem_one_percent = 0;
2526 uint32_t gc_heap::high_memory_load_th = 0;
2528 uint32_t gc_heap::m_high_memory_load_th;
2530 uint32_t gc_heap::v_high_memory_load_th;
2532 uint64_t gc_heap::total_physical_mem = 0;
2534 uint64_t gc_heap::entry_available_physical_mem = 0;
2536 size_t gc_heap::heap_hard_limit = 0;
2538 #ifdef BACKGROUND_GC
2539 GCEvent gc_heap::bgc_start_event;
2541 gc_mechanisms gc_heap::saved_bgc_settings;
2543 GCEvent gc_heap::background_gc_done_event;
2545 GCEvent gc_heap::ee_proceed_event;
2547 bool gc_heap::gc_can_use_concurrent = false;
2549 bool gc_heap::temp_disable_concurrent_p = false;
2551 uint32_t gc_heap::cm_in_progress = FALSE;
2553 BOOL gc_heap::dont_restart_ee_p = FALSE;
2555 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2557 GCEvent gc_heap::bgc_threads_sync_event;
2559 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2561 BOOL gc_heap::do_concurrent_p = FALSE;
2563 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2565 BOOL gc_heap::alloc_wait_event_p = FALSE;
2567 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2569 #endif //BACKGROUND_GC
2571 #ifndef MULTIPLE_HEAPS
2572 #ifdef SPINLOCK_HISTORY
2573 int gc_heap::spinlock_info_index = 0;
2574 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2575 #endif //SPINLOCK_HISTORY
2577 size_t gc_heap::fgn_last_alloc = 0;
2579 int gc_heap::generation_skip_ratio = 100;
2581 uint64_t gc_heap::loh_alloc_since_cg = 0;
2583 BOOL gc_heap::elevation_requested = FALSE;
2585 BOOL gc_heap::last_gc_before_oom = FALSE;
2587 BOOL gc_heap::sufficient_gen0_space_p = FALSE;
2589 #ifdef BACKGROUND_GC
2590 uint8_t* gc_heap::background_saved_lowest_address = 0;
2591 uint8_t* gc_heap::background_saved_highest_address = 0;
2592 uint8_t* gc_heap::next_sweep_obj = 0;
2593 uint8_t* gc_heap::current_sweep_pos = 0;
2594 exclusive_sync* gc_heap::bgc_alloc_lock;
2595 #endif //BACKGROUND_GC
2597 oom_history gc_heap::oom_info;
2599 fgm_history gc_heap::fgm_result;
2601 size_t gc_heap::allocated_since_last_gc = 0;
2603 BOOL gc_heap::ro_segments_in_range;
2605 size_t gc_heap::gen0_big_free_spaces = 0;
2607 uint8_t* gc_heap::ephemeral_low;
2609 uint8_t* gc_heap::ephemeral_high;
2611 uint8_t* gc_heap::lowest_address;
2613 uint8_t* gc_heap::highest_address;
2615 BOOL gc_heap::ephemeral_promotion;
2617 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2618 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2620 short* gc_heap::brick_table;
2622 uint32_t* gc_heap::card_table;
2625 uint32_t* gc_heap::card_bundle_table;
2626 #endif //CARD_BUNDLE
2628 uint8_t* gc_heap::gc_low;
2630 uint8_t* gc_heap::gc_high;
2632 uint8_t* gc_heap::demotion_low;
2634 uint8_t* gc_heap::demotion_high;
2636 BOOL gc_heap::demote_gen1_p = TRUE;
2638 uint8_t* gc_heap::last_gen1_pin_end;
2640 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2642 size_t gc_heap::etw_allocation_running_amount[2];
2644 int gc_heap::gc_policy = 0;
2646 size_t gc_heap::allocation_running_time;
2648 size_t gc_heap::allocation_running_amount;
2650 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2652 BOOL gc_heap::blocking_collection = FALSE;
2654 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2656 size_t gc_heap::time_bgc_last = 0;
2658 size_t gc_heap::mark_stack_tos = 0;
2660 size_t gc_heap::mark_stack_bos = 0;
2662 size_t gc_heap::mark_stack_array_length = 0;
2664 mark* gc_heap::mark_stack_array = 0;
2666 #if defined (_DEBUG) && defined (VERIFY_HEAP)
2667 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2668 #endif // defined (_DEBUG) && defined (VERIFY_HEAP)
2670 uint8_t* gc_heap::oldest_pinned_plug = 0;
2672 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2673 size_t gc_heap::num_pinned_objects = 0;
2674 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2676 #ifdef FEATURE_LOH_COMPACTION
2677 size_t gc_heap::loh_pinned_queue_tos = 0;
2679 size_t gc_heap::loh_pinned_queue_bos = 0;
2681 size_t gc_heap::loh_pinned_queue_length = 0;
2683 mark* gc_heap::loh_pinned_queue = 0;
2685 BOOL gc_heap::loh_compacted_p = FALSE;
2686 #endif //FEATURE_LOH_COMPACTION
2688 #ifdef BACKGROUND_GC
2690 EEThreadId gc_heap::bgc_thread_id;
2692 uint8_t* gc_heap::background_written_addresses [array_size+2];
2694 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2696 size_t gc_heap::bgc_overflow_count = 0;
2698 size_t gc_heap::bgc_begin_loh_size = 0;
2699 size_t gc_heap::end_loh_size = 0;
2701 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2703 size_t gc_heap::bgc_loh_size_increased = 0;
2705 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2707 size_t gc_heap::background_soh_alloc_count = 0;
2709 size_t gc_heap::background_loh_alloc_count = 0;
2711 uint8_t** gc_heap::background_mark_stack_tos = 0;
2713 uint8_t** gc_heap::background_mark_stack_array = 0;
2715 size_t gc_heap::background_mark_stack_array_length = 0;
2717 uint8_t* gc_heap::background_min_overflow_address =0;
2719 uint8_t* gc_heap::background_max_overflow_address =0;
2721 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2723 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2725 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2727 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2729 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2731 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2733 Thread* gc_heap::bgc_thread = 0;
2735 BOOL gc_heap::expanded_in_fgc = FALSE;
2737 uint8_t** gc_heap::c_mark_list = 0;
2739 size_t gc_heap::c_mark_list_length = 0;
2741 size_t gc_heap::c_mark_list_index = 0;
2743 gc_history_per_heap gc_heap::bgc_data_per_heap;
2745 BOOL gc_heap::bgc_thread_running;
2747 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2749 GCEvent gc_heap::gc_lh_block_event;
2751 #endif //BACKGROUND_GC
2754 uint8_t** gc_heap::mark_list;
2755 uint8_t** gc_heap::mark_list_index;
2756 uint8_t** gc_heap::mark_list_end;
2760 snoop_stats_data gc_heap::snoop_stat;
2761 #endif //SNOOP_STATS
2763 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2765 uint8_t* gc_heap::max_overflow_address = 0;
2767 uint8_t* gc_heap::shigh = 0;
2769 uint8_t* gc_heap::slow = MAX_PTR;
2771 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2773 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2775 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2777 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2779 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2781 BOOL gc_heap::use_bestfit = FALSE;
2783 uint8_t* gc_heap::bestfit_first_pin = 0;
2785 BOOL gc_heap::commit_end_of_seg = FALSE;
2787 size_t gc_heap::max_free_space_items = 0;
2789 size_t gc_heap::free_space_buckets = 0;
2791 size_t gc_heap::free_space_items = 0;
2793 int gc_heap::trimmed_free_space_index = 0;
2795 size_t gc_heap::total_ephemeral_plugs = 0;
2797 seg_free_spaces* gc_heap::bestfit_seg = 0;
2799 size_t gc_heap::total_ephemeral_size = 0;
2803 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2805 uint8_t** gc_heap::internal_root_array = 0;
2807 size_t gc_heap::internal_root_array_index = 0;
2809 BOOL gc_heap::heap_analyze_success = TRUE;
2811 uint8_t* gc_heap::current_obj = 0;
2812 size_t gc_heap::current_obj_size = 0;
2814 #endif //HEAP_ANALYZE
2816 #ifdef GC_CONFIG_DRIVEN
2817 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2818 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2819 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2820 #endif //GC_CONFIG_DRIVEN
2821 #endif //MULTIPLE_HEAPS
2823 no_gc_region_info gc_heap::current_no_gc_region_info;
2824 BOOL gc_heap::proceed_with_gc_p = FALSE;
2825 GCSpinLock gc_heap::gc_lock;
2827 size_t gc_heap::eph_gen_starts_size = 0;
2828 heap_segment* gc_heap::segment_standby_list;
2829 size_t gc_heap::last_gc_index = 0;
2830 #ifdef SEG_MAPPING_TABLE
2831 size_t gc_heap::min_segment_size = 0;
2832 size_t gc_heap::min_segment_size_shr = 0;
2833 #endif //SEG_MAPPING_TABLE
2834 size_t gc_heap::soh_segment_size = 0;
2835 size_t gc_heap::min_loh_segment_size = 0;
2836 size_t gc_heap::segment_info_size = 0;
2838 #ifdef GC_CONFIG_DRIVEN
2839 size_t gc_heap::time_init = 0;
2840 size_t gc_heap::time_since_init = 0;
2841 size_t gc_heap::compact_or_sweep_gcs[2];
2842 #endif //GC_CONFIG_DRIVEN
2844 #ifdef FEATURE_LOH_COMPACTION
2845 BOOL gc_heap::loh_compaction_always_p = FALSE;
2846 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2847 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2849 #endif //FEATURE_LOH_COMPACTION
2851 GCEvent gc_heap::full_gc_approach_event;
2853 GCEvent gc_heap::full_gc_end_event;
2855 uint32_t gc_heap::fgn_maxgen_percent = 0;
2857 uint32_t gc_heap::fgn_loh_percent = 0;
2859 #ifdef BACKGROUND_GC
2860 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2861 #endif //BACKGROUND_GC
2863 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2865 size_t gc_heap::full_gc_counts[gc_type_max];
2867 bool gc_heap::maxgen_size_inc_p = false;
2869 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2871 // Provisional mode related stuff.
2872 bool gc_heap::provisional_mode_triggered = false;
2873 bool gc_heap::pm_trigger_full_gc = false;
2874 size_t gc_heap::provisional_triggered_gc_count = 0;
2875 size_t gc_heap::provisional_off_gc_count = 0;
2876 size_t gc_heap::num_provisional_triggered = 0;
2877 bool gc_heap::pm_stress_on = false;
2880 BOOL gc_heap::heap_analyze_enabled = FALSE;
2881 #endif //HEAP_ANALYZE
2883 #ifndef MULTIPLE_HEAPS
2885 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2886 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2888 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2889 gc_history_per_heap gc_heap::gc_data_per_heap;
2890 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2892 uint8_t* gc_heap::alloc_allocated = 0;
2894 size_t gc_heap::allocation_quantum = CLR_SIZE;
2896 GCSpinLock gc_heap::more_space_lock_soh;
2897 GCSpinLock gc_heap::more_space_lock_loh;
2898 VOLATILE(int32_t) gc_heap::loh_alloc_thread_count = 0;
2900 #ifdef SYNCHRONIZATION_STATS
2901 unsigned int gc_heap::good_suspension = 0;
2902 unsigned int gc_heap::bad_suspension = 0;
2903 uint64_t gc_heap::total_msl_acquire = 0;
2904 unsigned int gc_heap::num_msl_acquired = 0;
2905 unsigned int gc_heap::num_high_msl_acquire = 0;
2906 unsigned int gc_heap::num_low_msl_acquire = 0;
2907 #endif //SYNCHRONIZATION_STATS
2909 size_t gc_heap::alloc_contexts_used = 0;
2910 size_t gc_heap::soh_allocation_no_gc = 0;
2911 size_t gc_heap::loh_allocation_no_gc = 0;
2912 bool gc_heap::no_gc_oom_p = false;
2913 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2915 #endif //MULTIPLE_HEAPS
2917 #ifndef MULTIPLE_HEAPS
2919 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2922 int gc_heap::gen0_must_clear_bricks = 0;
2923 #endif //FFIND_OBJECT
2925 #ifdef FEATURE_PREMORTEM_FINALIZATION
2926 CFinalize* gc_heap::finalize_queue = 0;
2927 #endif // FEATURE_PREMORTEM_FINALIZATION
2929 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2931 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2933 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2935 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2937 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2939 #endif // MULTIPLE_HEAPS
2941 /* end of per heap static initialization */
2943 /* end of static initialization */
2945 #ifndef DACCESS_COMPILE
2947 void gen_to_condemn_tuning::print (int heap_num)
2950 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2951 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2952 gc_condemn_reason_gen r_gen;
2953 for (int i = 0; i < gcrg_max; i++)
2955 r_gen = (gc_condemn_reason_gen)(i);
2956 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2958 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2960 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2961 gc_condemn_reason_condition r_condition;
2962 for (int i = 0; i < gcrc_max; i++)
2964 r_condition = (gc_condemn_reason_condition)(i);
2965 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2968 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2970 UNREFERENCED_PARAMETER(heap_num);
2974 void gc_generation_data::print (int heap_num, int gen_num)
2976 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2977 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2980 free_list_space_before, free_obj_space_before,
2982 free_list_space_after, free_obj_space_after,
2983 in, pinned_surv, npinned_surv,
2986 UNREFERENCED_PARAMETER(heap_num);
2987 UNREFERENCED_PARAMETER(gen_num);
2988 #endif //SIMPLE_DPRINTF && DT_LOG
2991 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2993 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2995 *mechanism |= mechanism_mask;
2996 *mechanism |= (1 << value);
2999 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
3000 dprintf (DT_LOG_0, ("setting %s: %s",
3002 (descr->descr)[value]));
3006 void gc_history_per_heap::print()
3008 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
3009 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
3011 gen_data[i].print (heap_index, i);
3014 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
3015 maxgen_size_info.free_list_allocated,
3016 maxgen_size_info.free_list_rejected,
3017 maxgen_size_info.end_seg_allocated,
3018 maxgen_size_info.condemned_allocated,
3019 maxgen_size_info.pinned_allocated,
3020 maxgen_size_info.pinned_allocated_advance,
3021 maxgen_size_info.running_free_list_efficiency,
3022 extra_gen0_committed));
3025 gc_mechanism_descr* descr = 0;
3027 for (int i = 0; i < max_mechanism_per_heap; i++)
3029 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
3033 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
3034 dprintf (DT_LOG_0, ("[%2d]%s%s",
3037 (descr->descr)[mechanism]));
3040 #endif //SIMPLE_DPRINTF && DT_LOG
3043 void gc_history_global::print()
3046 char str_settings[64];
3047 memset (str_settings, '|', sizeof (char) * 64);
3048 str_settings[max_global_mechanisms_count*2] = 0;
3050 for (int i = 0; i < max_global_mechanisms_count; i++)
3052 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
3055 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
3057 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
3058 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
3059 condemned_generation,
3060 str_gc_reasons[reason],
3061 str_gc_pause_modes[pause_mode],
3062 final_youngest_desired,
3063 gen0_reduction_count,
3068 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
3070 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
3071 FIRE_EVENT(GCPerHeapHistory_V3,
3072 (void *)(maxgen_size_info->free_list_allocated),
3073 (void *)(maxgen_size_info->free_list_rejected),
3074 (void *)(maxgen_size_info->end_seg_allocated),
3075 (void *)(maxgen_size_info->condemned_allocated),
3076 (void *)(maxgen_size_info->pinned_allocated),
3077 (void *)(maxgen_size_info->pinned_allocated_advance),
3078 maxgen_size_info->running_free_list_efficiency,
3079 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
3080 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3081 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3082 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3083 current_gc_data_per_heap->heap_index,
3084 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3085 (max_generation + 2),
3086 (uint32_t)(sizeof (gc_generation_data)),
3087 (void *)&(current_gc_data_per_heap->gen_data[0]));
3089 current_gc_data_per_heap->print();
3090 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3093 void gc_heap::fire_pevents()
3095 settings.record (&gc_data_global);
3096 gc_data_global.print();
3098 FIRE_EVENT(GCGlobalHeapHistory_V2,
3099 gc_data_global.final_youngest_desired,
3100 gc_data_global.num_heaps,
3101 gc_data_global.condemned_generation,
3102 gc_data_global.gen0_reduction_count,
3103 gc_data_global.reason,
3104 gc_data_global.global_mechanims_p,
3105 gc_data_global.pause_mode,
3106 gc_data_global.mem_pressure);
3108 #ifdef MULTIPLE_HEAPS
3109 for (int i = 0; i < gc_heap::n_heaps; i++)
3111 gc_heap* hp = gc_heap::g_heaps[i];
3112 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3113 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3116 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3117 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3122 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3128 case tuning_deciding_condemned_gen:
3129 case tuning_deciding_compaction:
3130 case tuning_deciding_expansion:
3131 case tuning_deciding_full_gc:
3133 ret = (!ephemeral_gen_fit_p (tp));
3136 case tuning_deciding_promote_ephemeral:
3138 size_t new_gen0size = approximate_new_allocation();
3139 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3141 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3142 heap_number, plan_ephemeral_size, new_gen0size));
3143 // If we were in no_gc_region we could have allocated a larger than normal segment,
3144 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3145 // ephemeral generations there, do an ephemeral promotion.
3146 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3157 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3165 case tuning_deciding_condemned_gen:
3167 dynamic_data* dd = dynamic_data_of (gen_number);
3168 float fragmentation_burden = 0;
3172 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3173 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3174 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3178 #ifndef MULTIPLE_HEAPS
3179 if (gen_number == max_generation)
3181 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3182 if (frag_ratio > 0.65)
3184 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3188 #endif //!MULTIPLE_HEAPS
3189 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3190 ret = (fr > dd_fragmentation_limit(dd));
3193 fragmentation_burden = (float)fr / generation_size (gen_number);
3194 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3196 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3197 heap_number, gen_number, dd_fragmentation (dd),
3198 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3199 fr, (int)(fragmentation_burden*100)));
3211 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3217 case tuning_deciding_condemned_gen:
3219 if (gen_number == max_generation)
3221 size_t est_maxgen_free = estimated_reclaim (gen_number);
3223 uint32_t num_heaps = 1;
3224 #ifdef MULTIPLE_HEAPS
3225 num_heaps = gc_heap::n_heaps;
3226 #endif //MULTIPLE_HEAPS
3228 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3229 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3230 ret = (est_maxgen_free >= min_frag_th);
3246 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3247 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3250 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3256 case tuning_deciding_condemned_gen:
3258 if (gen_number == max_generation)
3260 dynamic_data* dd = dynamic_data_of (gen_number);
3261 float est_frag_ratio = 0;
3262 if (dd_current_size (dd) == 0)
3266 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3272 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3275 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3276 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3279 dd_current_size (dd),
3280 dd_fragmentation (dd),
3281 (int)(est_frag_ratio*100),
3284 uint32_t num_heaps = 1;
3286 #ifdef MULTIPLE_HEAPS
3287 num_heaps = gc_heap::n_heaps;
3288 #endif //MULTIPLE_HEAPS
3289 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3290 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3291 ret = (est_frag >= min_frag_th);
3308 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3314 case tuning_deciding_condemned_gen:
3316 /* promote into max-generation if the card table has too many
3317 * generation faults besides the n -> 0
3319 ret = (generation_skip_ratio < 30);
3331 in_range_for_segment(uint8_t* add, heap_segment* seg)
3333 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3336 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3337 // The array we allocate is organized as follows:
3338 // 0th element is the address of the last array we allocated.
3339 // starting from the 1st element are the segment addresses, that's
3340 // what buckets() returns.
3353 bk* buckets() { return (slots + 1); }
3354 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3357 static sorted_table* make_sorted_table ();
3358 BOOL insert (uint8_t* add, size_t val);;
3359 size_t lookup (uint8_t*& add);
3360 void remove (uint8_t* add);
3362 void delete_sorted_table();
3363 void delete_old_slots();
3364 void enqueue_old_slot(bk* sl);
3365 BOOL ensure_space_for_insert();
3369 sorted_table::make_sorted_table ()
3373 // allocate one more bk to store the older slot address.
3374 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3378 res->slots = (bk*)(res + 1);
3385 sorted_table::delete_sorted_table()
3387 if (slots != (bk*)(this+1))
3395 sorted_table::delete_old_slots()
3397 uint8_t* sl = (uint8_t*)old_slots;
3401 sl = last_slot ((bk*)sl);
3407 sorted_table::enqueue_old_slot(bk* sl)
3409 last_slot (sl) = (uint8_t*)old_slots;
3415 sorted_table::lookup (uint8_t*& add)
3417 ptrdiff_t high = (count-1);
3421 bk* buck = buckets();
3424 mid = ((low + high)/2);
3426 if (buck[ti].add > add)
3428 if ((ti > 0) && (buck[ti-1].add <= add))
3430 add = buck[ti-1].add;
3431 return buck[ti - 1].val;
3437 if (buck[ti+1].add > add)
3440 return buck[ti].val;
3450 sorted_table::ensure_space_for_insert()
3454 size = (size * 3)/2;
3455 assert((size * sizeof (bk)) > 0);
3456 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3461 last_slot (res) = 0;
3462 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3463 bk* last_old_slots = slots;
3465 if (last_old_slots != (bk*)(this + 1))
3466 enqueue_old_slot (last_old_slots);
3472 sorted_table::insert (uint8_t* add, size_t val)
3474 //grow if no more room
3475 assert (count < size);
3478 ptrdiff_t high = (count-1);
3482 bk* buck = buckets();
3485 mid = ((low + high)/2);
3487 if (buck[ti].add > add)
3489 if ((ti == 0) || (buck[ti-1].add <= add))
3491 // found insertion point
3492 for (ptrdiff_t k = count; k > ti;k--)
3494 buck [k] = buck [k-1];
3505 if (buck[ti+1].add > add)
3507 //found the insertion point
3508 for (ptrdiff_t k = count; k > ti+1;k--)
3510 buck [k] = buck [k-1];
3512 buck[ti+1].add = add;
3513 buck[ti+1].val = val;
3525 sorted_table::remove (uint8_t* add)
3527 ptrdiff_t high = (count-1);
3531 bk* buck = buckets();
3534 mid = ((low + high)/2);
3536 if (buck[ti].add > add)
3538 if (buck[ti-1].add <= add)
3540 // found the guy to remove
3541 for (ptrdiff_t k = ti; k < count; k++)
3542 buck[k-1] = buck[k];
3550 if (buck[ti+1].add > add)
3552 // found the guy to remove
3553 for (ptrdiff_t k = ti+1; k < count; k++)
3554 buck[k-1] = buck[k];
3565 sorted_table::clear()
3568 buckets()[0].add = MAX_PTR;
3570 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3572 #ifdef SEG_MAPPING_TABLE
3573 #ifdef GROWABLE_SEG_MAPPING_TABLE
3575 uint8_t* align_on_segment (uint8_t* add)
3577 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3581 uint8_t* align_lower_segment (uint8_t* add)
3583 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3586 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3588 from = align_lower_segment (from);
3589 end = align_on_segment (end);
3590 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3591 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3594 // for seg_mapping_table we want it to start from a pointer sized address.
3596 size_t align_for_seg_mapping_table (size_t size)
3598 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3602 size_t seg_mapping_word_of (uint8_t* add)
3604 return (size_t)add >> gc_heap::min_segment_size_shr;
3606 #else //GROWABLE_SEG_MAPPING_TABLE
3607 BOOL seg_mapping_table_init()
3610 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3612 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3615 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3616 seg_mapping_table = new seg_mapping[num_entries];
3618 if (seg_mapping_table)
3620 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3621 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3622 num_entries, (num_entries * sizeof (seg_mapping))));
3627 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3628 num_entries, (num_entries * sizeof (seg_mapping))));
3632 #endif //GROWABLE_SEG_MAPPING_TABLE
3634 #ifdef FEATURE_BASICFREEZE
3636 size_t ro_seg_begin_index (heap_segment* seg)
3638 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3639 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3644 size_t ro_seg_end_index (heap_segment* seg)
3646 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3647 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3651 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3653 #ifdef GROWABLE_SEG_MAPPING_TABLE
3654 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3656 #endif //GROWABLE_SEG_MAPPING_TABLE
3658 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3659 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3662 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3664 UNREFERENCED_PARAMETER(seg);
3666 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3667 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3668 // remove the flag if none lands in this range.
3672 heap_segment* ro_segment_lookup (uint8_t* o)
3674 uint8_t* ro_seg_start = o;
3675 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3677 if (ro_seg_start && in_range_for_segment (o, seg))
3683 #endif //FEATURE_BASICFREEZE
3685 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3687 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3688 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3689 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3690 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3691 seg_mapping* end_entry = &seg_mapping_table[end_index];
3693 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3694 seg, begin_index, heap_segment_reserved (seg), end_index));
3696 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3697 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3698 end_index, (seg_mapping_table[end_index].boundary + 1)));
3700 #ifdef MULTIPLE_HEAPS
3701 #ifdef SIMPLE_DPRINTF
3702 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3703 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3704 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3705 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3706 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3707 #endif //SIMPLE_DPRINTF
3708 assert (end_entry->boundary == 0);
3709 assert (end_entry->h0 == 0);
3711 assert (begin_entry->h1 == 0);
3712 begin_entry->h1 = hp;
3714 UNREFERENCED_PARAMETER(hp);
3715 #endif //MULTIPLE_HEAPS
3717 end_entry->boundary = (uint8_t*)seg_end;
3719 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3720 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3721 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3722 end_entry->seg0 = seg;
3724 // for every entry inbetween we need to set its heap too.
3725 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3727 assert (seg_mapping_table[entry_index].boundary == 0);
3728 #ifdef MULTIPLE_HEAPS
3729 assert (seg_mapping_table[entry_index].h0 == 0);
3730 seg_mapping_table[entry_index].h1 = hp;
3731 #endif //MULTIPLE_HEAPS
3732 seg_mapping_table[entry_index].seg1 = seg;
3735 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3736 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3737 end_index, (seg_mapping_table[end_index].boundary + 1)));
3738 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3739 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3740 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3741 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3742 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3743 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3744 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3747 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3749 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3750 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3751 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3752 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3753 seg_mapping* end_entry = &seg_mapping_table[end_index];
3754 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3755 seg, begin_index, heap_segment_reserved (seg), end_index));
3757 assert (end_entry->boundary == (uint8_t*)seg_end);
3758 end_entry->boundary = 0;
3760 #ifdef MULTIPLE_HEAPS
3761 gc_heap* hp = heap_segment_heap (seg);
3762 assert (end_entry->h0 == hp);
3764 assert (begin_entry->h1 == hp);
3765 begin_entry->h1 = 0;
3766 #endif //MULTIPLE_HEAPS
3768 assert (begin_entry->seg1 != 0);
3769 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3770 end_entry->seg0 = 0;
3772 // for every entry inbetween we need to reset its heap too.
3773 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3775 assert (seg_mapping_table[entry_index].boundary == 0);
3776 #ifdef MULTIPLE_HEAPS
3777 assert (seg_mapping_table[entry_index].h0 == 0);
3778 assert (seg_mapping_table[entry_index].h1 == hp);
3779 seg_mapping_table[entry_index].h1 = 0;
3780 #endif //MULTIPLE_HEAPS
3781 seg_mapping_table[entry_index].seg1 = 0;
3784 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3785 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3786 end_index, (seg_mapping_table[end_index].boundary + 1)));
3787 #ifdef MULTIPLE_HEAPS
3788 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3789 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3790 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3791 #endif //MULTIPLE_HEAPS
3794 #ifdef MULTIPLE_HEAPS
3796 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3798 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3799 seg_mapping* entry = &seg_mapping_table[index];
3801 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3803 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3804 o, index, (entry->boundary + 1),
3805 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3806 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3809 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3810 #ifdef FEATURE_BASICFREEZE
3811 if ((size_t)seg & ro_in_entry)
3812 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3813 #endif //FEATURE_BASICFREEZE
3817 if (in_range_for_segment (o, seg))
3819 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3823 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3824 seg, (uint8_t*)heap_segment_allocated (seg), o));
3829 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3836 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3838 #ifdef GROWABLE_SEG_MAPPING_TABLE
3839 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3841 #endif //GROWABLE_SEG_MAPPING_TABLE
3843 return seg_mapping_table_heap_of_worker (o);
3846 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3848 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3849 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3851 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3853 return seg_mapping_table_heap_of_worker (o);
3855 #endif //MULTIPLE_HEAPS
3857 // Only returns a valid seg if we can actually find o on the seg.
3858 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3860 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3861 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3862 #ifdef FEATURE_BASICFREEZE
3863 return ro_segment_lookup (o);
3866 #endif //FEATURE_BASICFREEZE
3867 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3869 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3870 seg_mapping* entry = &seg_mapping_table[index];
3872 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3873 o, index, (entry->boundary + 1),
3874 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3876 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3877 #ifdef FEATURE_BASICFREEZE
3878 if ((size_t)seg & ro_in_entry)
3879 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3880 #endif //FEATURE_BASICFREEZE
3884 if (in_range_for_segment (o, seg))
3886 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3890 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3891 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3897 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3900 #ifdef FEATURE_BASICFREEZE
3901 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3902 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3903 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3904 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3905 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3906 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3909 seg = ro_segment_lookup (o);
3910 if (seg && !in_range_for_segment (o, seg))
3913 #endif //FEATURE_BASICFREEZE
3917 #endif //SEG_MAPPING_TABLE
3919 size_t gcard_of ( uint8_t*);
3921 #define memref(i) *(uint8_t**)(i)
3924 #define GC_MARKED (size_t)0x1
3925 #define slot(i, j) ((uint8_t**)(i))[j+1]
3927 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3929 class CObjectHeader : public Object
3933 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3934 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3935 // by Redhawk's version of Object.
3936 uint32_t GetNumComponents()
3938 return ((ArrayBase *)this)->GetNumComponents();
3941 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3943 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3948 MethodTable * pMT = GetMethodTable();
3950 _ASSERTE(pMT->SanityCheck());
3952 bool noRangeChecks =
3953 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3955 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3958 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3959 if (!fSmallObjectHeapPtr)
3960 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3962 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3965 #ifdef FEATURE_STRUCTALIGN
3966 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3967 #endif // FEATURE_STRUCTALIGN
3969 #ifdef FEATURE_64BIT_ALIGNMENT
3970 if (pMT->RequiresAlign8())
3972 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3974 #endif // FEATURE_64BIT_ALIGNMENT
3977 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3978 g_theGCHeap->ValidateObjectMember(this);
3980 if (fSmallObjectHeapPtr)
3982 #ifdef FEATURE_BASICFREEZE
3983 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3985 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3990 void ValidatePromote(ScanContext *sc, uint32_t flags)
3992 UNREFERENCED_PARAMETER(sc);
3993 UNREFERENCED_PARAMETER(flags);
3998 void ValidateHeap(Object *from, BOOL bDeep)
4000 UNREFERENCED_PARAMETER(from);
4002 Validate(bDeep, FALSE);
4005 #endif //FEATURE_REDHAWK || BUILD_AS_STANDALONE
4009 // Header Status Information
4012 MethodTable *GetMethodTable() const
4014 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
4019 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
4022 BOOL IsMarked() const
4024 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
4029 assert (!(gc_heap::settings.concurrent));
4030 GetHeader()->SetGCBit();
4033 BOOL IsPinned() const
4035 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
4040 RawSetMethodTable( GetMethodTable() );
4043 CGCDesc *GetSlotMap ()
4045 assert (GetMethodTable()->ContainsPointers());
4046 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
4049 void SetFree(size_t size)
4051 assert (size >= free_object_base_size);
4053 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
4054 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
4056 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
4058 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
4059 *numComponentsPtr = size - free_object_base_size;
4061 //This introduces a bug in the free list management.
4062 //((void**) this)[-1] = 0; // clear the sync block,
4063 assert (*numComponentsPtr >= 0);
4064 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4065 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4066 #endif //VERIFY_HEAP
4071 size_t size = free_object_base_size - plug_skew;
4073 // since we only need to clear 2 ptr size, we do it manually
4074 PTR_PTR m = (PTR_PTR) this;
4075 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4079 BOOL IsFree () const
4081 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4084 #ifdef FEATURE_STRUCTALIGN
4085 int GetRequiredAlignment () const
4087 return GetMethodTable()->GetRequiredAlignment();
4089 #endif // FEATURE_STRUCTALIGN
4091 BOOL ContainsPointers() const
4093 return GetMethodTable()->ContainsPointers();
4096 #ifdef COLLECTIBLE_CLASS
4097 BOOL Collectible() const
4099 return GetMethodTable()->Collectible();
4102 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4104 MethodTable *pMethodTable = GetMethodTable();
4105 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4107 #endif //COLLECTIBLE_CLASS
4109 Object* GetObjectBase() const
4111 return (Object*) this;
4115 #define header(i) ((CObjectHeader*)(i))
4117 #define free_list_slot(x) ((uint8_t**)(x))[2]
4118 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4119 #define UNDO_EMPTY ((uint8_t*)1)
4123 void set_plug_padded (uint8_t* node)
4125 header(node)->SetMarked();
4128 void clear_plug_padded (uint8_t* node)
4130 header(node)->ClearMarked();
4133 BOOL is_plug_padded (uint8_t* node)
4135 return header(node)->IsMarked();
4138 inline void set_plug_padded (uint8_t* node){}
4139 inline void clear_plug_padded (uint8_t* node){}
4141 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4142 #endif //SHORT_PLUGS
4145 inline size_t unused_array_size(uint8_t * p)
4147 assert(((CObjectHeader*)p)->IsFree());
4149 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4150 return free_object_base_size + *numComponentsPtr;
4153 heap_segment* heap_segment_rw (heap_segment* ns)
4155 if ((ns == 0) || !heap_segment_read_only_p (ns))
4163 ns = heap_segment_next (ns);
4164 } while ((ns != 0) && heap_segment_read_only_p (ns));
4169 //returns the next non ro segment.
4170 heap_segment* heap_segment_next_rw (heap_segment* seg)
4172 heap_segment* ns = heap_segment_next (seg);
4173 return heap_segment_rw (ns);
4176 // returns the segment before seg.
4177 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4179 assert (begin != 0);
4180 heap_segment* prev = begin;
4181 heap_segment* current = heap_segment_next_rw (begin);
4183 while (current && current != seg)
4186 current = heap_segment_next_rw (current);
4199 // returns the segment before seg.
4200 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4202 assert (begin != 0);
4203 heap_segment* prev = begin;
4204 heap_segment* current = heap_segment_next (begin);
4206 while (current && current != seg)
4209 current = heap_segment_next (current);
4222 heap_segment* heap_segment_in_range (heap_segment* ns)
4224 if ((ns == 0) || heap_segment_in_range_p (ns))
4232 ns = heap_segment_next (ns);
4233 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4238 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4240 heap_segment* ns = heap_segment_next (seg);
4241 return heap_segment_in_range (ns);
4246 uint8_t* memory_base;
4251 imemory_data *initial_memory;
4252 imemory_data *initial_normal_heap; // points into initial_memory_array
4253 imemory_data *initial_large_heap; // points into initial_memory_array
4255 size_t block_size_normal;
4256 size_t block_size_large;
4258 size_t block_count; // # of blocks in each
4259 size_t current_block_normal;
4260 size_t current_block_large;
4269 size_t allocation_pattern;
4270 } initial_memory_details;
4272 initial_memory_details memory_details;
4274 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4276 BOOL reserve_success = FALSE;
4278 // should only be called once
4279 assert (memory_details.initial_memory == 0);
4281 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4282 if (memory_details.initial_memory == 0)
4284 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4288 memory_details.initial_normal_heap = memory_details.initial_memory;
4289 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4290 memory_details.block_size_normal = normal_size;
4291 memory_details.block_size_large = large_size;
4292 memory_details.block_count = num_heaps;
4294 memory_details.current_block_normal = 0;
4295 memory_details.current_block_large = 0;
4297 g_gc_lowest_address = MAX_PTR;
4298 g_gc_highest_address = 0;
4300 if (((size_t)MAX_PTR - large_size) < normal_size)
4302 // we are already overflowing with just one heap.
4303 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4307 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4309 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4313 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4315 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4316 if (allatonce_block)
4318 g_gc_lowest_address = allatonce_block;
4319 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4320 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4322 for(size_t i = 0; i < memory_details.block_count; i++)
4324 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4325 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4326 (memory_details.block_count*normal_size) + (i*large_size);
4327 reserve_success = TRUE;
4332 // try to allocate 2 blocks
4335 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4338 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4341 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4342 g_gc_lowest_address = min(b1,b2);
4343 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4344 b2 + memory_details.block_count*large_size);
4345 for(size_t i = 0; i < memory_details.block_count; i++)
4347 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4348 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4349 reserve_success = TRUE;
4354 // b2 allocation failed, we'll go on to try allocating each block.
4355 // We could preserve the b1 alloc, but code complexity increases
4356 virtual_free (b1, memory_details.block_count * normal_size);
4360 if ((b2==NULL) && ( memory_details.block_count > 1))
4362 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4364 imemory_data *current_block = memory_details.initial_memory;
4365 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4367 size_t block_size = ((i < memory_details.block_count) ?
4368 memory_details.block_size_normal :
4369 memory_details.block_size_large);
4370 current_block->memory_base =
4371 (uint8_t*)virtual_alloc (block_size);
4372 if (current_block->memory_base == 0)
4374 // Free the blocks that we've allocated so far
4375 current_block = memory_details.initial_memory;
4376 for(size_t j = 0; j < i; j++, current_block++){
4377 if (current_block->memory_base != 0){
4378 block_size = ((j < memory_details.block_count) ?
4379 memory_details.block_size_normal :
4380 memory_details.block_size_large);
4381 virtual_free (current_block->memory_base , block_size);
4384 reserve_success = FALSE;
4389 if (current_block->memory_base < g_gc_lowest_address)
4390 g_gc_lowest_address = current_block->memory_base;
4391 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4392 g_gc_highest_address = (current_block->memory_base + block_size);
4394 reserve_success = TRUE;
4399 return reserve_success;
4402 void destroy_initial_memory()
4404 if (memory_details.initial_memory != NULL)
4406 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4408 virtual_free(memory_details.initial_memory[0].memory_base,
4409 memory_details.block_count*(memory_details.block_size_normal +
4410 memory_details.block_size_large));
4412 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4414 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4415 memory_details.block_count*memory_details.block_size_normal);
4417 virtual_free (memory_details.initial_large_heap[0].memory_base,
4418 memory_details.block_count*memory_details.block_size_large);
4422 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4423 imemory_data *current_block = memory_details.initial_memory;
4424 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4426 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4427 memory_details.block_size_large;
4428 if (current_block->memory_base != NULL)
4430 virtual_free (current_block->memory_base, block_size);
4435 delete [] memory_details.initial_memory;
4436 memory_details.initial_memory = NULL;
4437 memory_details.initial_normal_heap = NULL;
4438 memory_details.initial_large_heap = NULL;
4442 void* next_initial_memory (size_t size)
4444 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4447 if ((size != memory_details.block_size_normal) ||
4448 ((memory_details.current_block_normal == memory_details.block_count) &&
4449 (memory_details.block_size_normal == memory_details.block_size_large)))
4451 // If the block sizes are the same, flow block requests from normal to large
4452 assert (memory_details.current_block_large < memory_details.block_count);
4453 assert (memory_details.initial_large_heap != 0);
4455 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4456 memory_details.current_block_large++;
4460 assert (memory_details.current_block_normal < memory_details.block_count);
4461 assert (memory_details.initial_normal_heap != NULL);
4463 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4464 memory_details.current_block_normal++;
4470 heap_segment* get_initial_segment (size_t size, int h_number)
4472 void* mem = next_initial_memory (size);
4473 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4478 void* virtual_alloc (size_t size)
4480 size_t requested_size = size;
4482 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4484 gc_heap::reserved_memory_limit =
4485 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4486 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4492 uint32_t flags = VirtualReserveFlags::None;
4493 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4494 if (virtual_alloc_hardware_write_watch)
4496 flags = VirtualReserveFlags::WriteWatch;
4498 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4499 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4500 void *aligned_mem = prgmem;
4502 // We don't want (prgmem + size) to be right at the end of the address space
4503 // because we'd have to worry about that everytime we do (address + size).
4504 // We also want to make sure that we leave loh_size_threshold at the end
4505 // so we allocate a small object we don't need to worry about overflow there
4506 // when we do alloc_ptr+size.
4509 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4511 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4513 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4514 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4515 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4523 gc_heap::reserved_memory += requested_size;
4526 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4527 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4532 void virtual_free (void* add, size_t size)
4534 GCToOSInterface::VirtualRelease (add, size);
4535 gc_heap::reserved_memory -= size;
4536 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4537 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4540 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4542 size_t seg_size, initial_seg_size;
4546 initial_seg_size = INITIAL_ALLOC;
4547 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4551 initial_seg_size = LHEAP_ALLOC;
4552 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4555 #ifdef MULTIPLE_HEAPS
4560 if (g_num_processors > 4)
4561 initial_seg_size /= 2;
4562 if (g_num_processors > 8)
4563 initial_seg_size /= 2;
4565 #endif //MULTIPLE_HEAPS
4567 // if seg_size is small but not 0 (0 is default if config not set)
4568 // then set the segment to the minimum size
4569 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4571 // if requested size is between 1 byte and 4MB, use min
4572 if ((seg_size >> 1) && !(seg_size >> 22))
4573 seg_size = 1024*1024*4;
4575 seg_size = initial_seg_size;
4578 #ifdef SEG_MAPPING_TABLE
4580 seg_size = round_up_power2 (seg_size);
4582 seg_size = round_down_power2 (seg_size);
4584 #endif //SEG_MAPPING_TABLE
4590 gc_heap::compute_new_ephemeral_size()
4592 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4593 size_t padding_size = 0;
4595 for (int i = 0; i <= eph_gen_max; i++)
4597 dynamic_data* dd = dynamic_data_of (i);
4598 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4599 #ifdef RESPECT_LARGE_ALIGNMENT
4600 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4601 #endif //RESPECT_LARGE_ALIGNMENT
4602 #ifdef FEATURE_STRUCTALIGN
4603 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4604 #endif //FEATURE_STRUCTALIGN
4607 padding_size += dd_padding_size (dd);
4608 #endif //SHORT_PLUGS
4611 total_ephemeral_size += eph_gen_starts_size;
4613 #ifdef RESPECT_LARGE_ALIGNMENT
4614 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4615 generation_plan_allocation_start (generation_of (max_generation-1));
4616 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4617 #endif //RESPECT_LARGE_ALIGNMENT
4620 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4621 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4622 #endif //SHORT_PLUGS
4624 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4625 total_ephemeral_size,
4626 padding_size, (total_ephemeral_size - padding_size)));
4630 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4634 gc_heap::soh_get_segment_to_expand()
4636 size_t size = soh_segment_size;
4638 ordered_plug_indices_init = FALSE;
4639 use_bestfit = FALSE;
4641 //compute the size of the new ephemeral heap segment.
4642 compute_new_ephemeral_size();
4644 if ((settings.pause_mode != pause_low_latency) &&
4645 (settings.pause_mode != pause_no_gc)
4646 #ifdef BACKGROUND_GC
4647 && (!recursive_gc_sync::background_running_p())
4648 #endif //BACKGROUND_GC
4651 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4652 generation_allocator (generation_of (max_generation)));
4653 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4655 // try to find one in the gen 2 segment list, search backwards because the first segments
4656 // tend to be more compact than the later ones.
4657 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4659 PREFIX_ASSUME(fseg != NULL);
4661 #ifdef SEG_REUSE_STATS
4663 #endif //SEG_REUSE_STATS
4665 heap_segment* seg = ephemeral_heap_segment;
4666 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4668 #ifdef SEG_REUSE_STATS
4670 #endif //SEG_REUSE_STATS
4672 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4674 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4675 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4676 if (settings.condemned_generation == max_generation)
4680 build_ordered_free_spaces (seg);
4681 dprintf (GTC_LOG, ("can use best fit"));
4684 #ifdef SEG_REUSE_STATS
4685 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4686 settings.condemned_generation, try_reuse));
4687 #endif //SEG_REUSE_STATS
4688 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4693 #ifdef SEG_REUSE_STATS
4694 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4695 settings.condemned_generation, try_reuse));
4696 #endif //SEG_REUSE_STATS
4697 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4699 // If we return 0 here, the allocator will think since we are short on end
4700 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4701 // is set we should acquire a new seg instead, that way we wouldn't be short.
4702 // The real solution, of course, is to actually implement seg reuse in gen1.
4703 if (settings.pause_mode != pause_sustained_low_latency)
4705 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4706 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4714 heap_segment* result = get_segment (size, FALSE);
4718 #ifdef BACKGROUND_GC
4719 if (current_c_gc_state == c_gc_state_planning)
4721 // When we expand heap during bgc sweep, we set the seg to be swept so
4722 // we'll always look at cards for objects on the new segment.
4723 result->flags |= heap_segment_flags_swept;
4725 #endif //BACKGROUND_GC
4727 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4728 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4729 gc_etw_segment_small_object_heap);
4732 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4736 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4740 #ifdef MULTIPLE_HEAPS
4741 heap_segment_heap (result) = this;
4742 #endif //MULTIPLE_HEAPS
4745 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4750 #pragma warning(default:4706)
4753 //returns 0 in case of allocation failure
4755 gc_heap::get_segment (size_t size, BOOL loh_p)
4757 if (heap_hard_limit)
4760 heap_segment* result = 0;
4762 if (segment_standby_list != 0)
4764 result = segment_standby_list;
4765 heap_segment* last = 0;
4768 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4769 if ((hs >= size) && ((hs / 2) < size))
4771 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4774 heap_segment_next (last) = heap_segment_next (result);
4778 segment_standby_list = heap_segment_next (result);
4785 result = heap_segment_next (result);
4792 init_heap_segment (result);
4793 #ifdef BACKGROUND_GC
4794 if (should_commit_mark_array())
4796 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4797 if (!commit_mark_array_new_seg (__this, result))
4799 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4800 // If we can't use it we need to thread it back.
4801 if (segment_standby_list != 0)
4803 heap_segment_next (result) = segment_standby_list;
4804 segment_standby_list = result;
4808 segment_standby_list = result;
4814 #endif //BACKGROUND_GC
4816 #ifdef SEG_MAPPING_TABLE
4818 seg_mapping_table_add_segment (result, __this);
4819 #endif //SEG_MAPPING_TABLE
4824 #ifndef SEG_MAPPING_TABLE
4825 if (!seg_table->ensure_space_for_insert ())
4827 #endif //SEG_MAPPING_TABLE
4828 void* mem = virtual_alloc (size);
4831 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4835 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4841 if (mem < g_gc_lowest_address)
4843 start = (uint8_t*)mem;
4847 start = (uint8_t*)g_gc_lowest_address;
4850 if (((uint8_t*)mem + size) > g_gc_highest_address)
4852 end = (uint8_t*)mem + size;
4856 end = (uint8_t*)g_gc_highest_address;
4859 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4861 virtual_free (mem, size);
4867 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4868 virtual_free (mem, size);
4873 #ifdef SEG_MAPPING_TABLE
4874 seg_mapping_table_add_segment (result, __this);
4875 #else //SEG_MAPPING_TABLE
4876 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4877 #endif //SEG_MAPPING_TABLE
4881 #ifdef BACKGROUND_GC
4884 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4885 settings.gc_index, current_bgc_state,
4887 bgc_verify_mark_array_cleared (result);
4889 #endif //BACKGROUND_GC
4891 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4895 void release_segment (heap_segment* sg)
4897 ptrdiff_t delta = 0;
4898 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4899 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4902 heap_segment* gc_heap::get_segment_for_loh (size_t size
4903 #ifdef MULTIPLE_HEAPS
4905 #endif //MULTIPLE_HEAPS
4908 #ifndef MULTIPLE_HEAPS
4910 #endif //MULTIPLE_HEAPS
4911 heap_segment* res = hp->get_segment (size, TRUE);
4914 #ifdef MULTIPLE_HEAPS
4915 heap_segment_heap (res) = hp;
4916 #endif //MULTIPLE_HEAPS
4917 res->flags |= heap_segment_flags_loh;
4919 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4921 GCToEEInterface::DiagUpdateGenerationBounds();
4923 #ifdef MULTIPLE_HEAPS
4924 hp->thread_loh_segment (res);
4926 thread_loh_segment (res);
4927 #endif //MULTIPLE_HEAPS
4933 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4935 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4937 while (heap_segment_next_rw (seg))
4938 seg = heap_segment_next_rw (seg);
4939 heap_segment_next (seg) = new_seg;
4943 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4945 *did_full_compact_gc = FALSE;
4946 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4948 //access to get_segment needs to be serialized
4949 add_saved_spinlock_info (true, me_release, mt_get_large_seg);
4950 leave_spin_lock (&more_space_lock_loh);
4951 enter_spin_lock (&gc_heap::gc_lock);
4952 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4953 // if a GC happened between here and before we ask for a segment in
4954 // get_large_segment, we need to count that GC.
4955 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4957 if (current_full_compact_gc_count > last_full_compact_gc_count)
4959 *did_full_compact_gc = TRUE;
4962 heap_segment* res = get_segment_for_loh (size
4963 #ifdef MULTIPLE_HEAPS
4965 #endif //MULTIPLE_HEAPS
4968 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4969 leave_spin_lock (&gc_heap::gc_lock);
4970 enter_spin_lock (&more_space_lock_loh);
4971 add_saved_spinlock_info (true, me_acquire, mt_get_large_seg);
4977 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4979 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4980 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4982 if (region_size != 0 )
4984 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4986 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4994 #ifdef MULTIPLE_HEAPS
4997 #pragma warning(disable:4035)
4998 static ptrdiff_t get_cycle_count()
5002 #pragma warning(default:4035)
5003 #elif defined(__GNUC__)
5004 static ptrdiff_t get_cycle_count()
5008 __asm__ __volatile__
5009 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5013 #error Unknown compiler
5015 #elif defined(_TARGET_AMD64_)
5017 extern "C" uint64_t __rdtsc();
5018 #pragma intrinsic(__rdtsc)
5019 static ptrdiff_t get_cycle_count()
5021 return (ptrdiff_t)__rdtsc();
5023 #elif defined(__GNUC__)
5024 static ptrdiff_t get_cycle_count()
5028 __asm__ __volatile__
5029 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5030 return (cyclesHi << 32) | cycles;
5033 extern "C" ptrdiff_t get_cycle_count(void);
5035 #elif defined(_TARGET_ARM_)
5036 static ptrdiff_t get_cycle_count()
5038 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5039 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5040 // all buffer access times being reported as equal in access_time().
5043 #elif defined(_TARGET_ARM64_)
5044 static ptrdiff_t get_cycle_count()
5046 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5047 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5048 // all buffer access times being reported as equal in access_time().
5052 #error NYI platform: get_cycle_count
5053 #endif //_TARGET_X86_
5058 static uint8_t* sniff_buffer;
5059 static unsigned n_sniff_buffers;
5060 static unsigned cur_sniff_index;
5062 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5063 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5064 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5065 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5066 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5067 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5069 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5071 ptrdiff_t start_cycles = get_cycle_count();
5072 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5073 assert (sniff == 0);
5074 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5075 // add sniff here just to defeat the optimizer
5076 elapsed_cycles += sniff;
5077 return (int) elapsed_cycles;
5081 static BOOL init(int n_heaps)
5083 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5084 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5086 n_sniff_buffers = n_heaps*2+1;
5087 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5088 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5089 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5094 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5095 if (sniff_buffer == 0)
5097 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5100 //can not enable gc numa aware, force all heaps to be in
5101 //one numa node by filling the array with all 0s
5102 if (!GCToOSInterface::CanEnableGCNumaAware())
5103 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5108 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5110 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5112 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5113 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5114 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5115 // MAX_SUPPORTED_CPUS GC threads.
5116 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5120 static void mark_heap(int heap_number)
5122 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5125 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5126 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5129 static int select_heap(alloc_context* acontext, int /*hint*/)
5131 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5133 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5134 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5136 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5137 sniff_index %= n_sniff_buffers;
5140 int best_access_time = 1000*1000*1000;
5141 int second_best_access_time = best_access_time;
5143 uint8_t *l_sniff_buffer = sniff_buffer;
5144 unsigned l_n_sniff_buffers = n_sniff_buffers;
5145 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5147 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5148 if (this_access_time < best_access_time)
5150 second_best_access_time = best_access_time;
5151 best_access_time = this_access_time;
5152 best_heap = heap_number;
5154 else if (this_access_time < second_best_access_time)
5156 second_best_access_time = this_access_time;
5160 if (best_access_time*2 < second_best_access_time)
5162 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5164 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5168 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5174 static bool can_find_heap_fast()
5176 return GCToOSInterface::CanGetCurrentProcessorNumber();
5179 static uint16_t find_proc_no_from_heap_no(int heap_number)
5181 return heap_no_to_proc_no[heap_number];
5184 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5186 heap_no_to_proc_no[heap_number] = proc_no;
5189 static uint16_t find_numa_node_from_heap_no(int heap_number)
5191 return heap_no_to_numa_node[heap_number];
5194 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5196 heap_no_to_numa_node[heap_number] = numa_node;
5199 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5201 return heap_no_to_cpu_group[heap_number];
5204 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5206 heap_no_to_cpu_group[heap_number] = group_number;
5209 static uint16_t find_group_proc_from_heap_no(int heap_number)
5211 return heap_no_to_group_proc[heap_number];
5214 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5216 heap_no_to_group_proc[heap_number] = group_proc;
5219 static void init_numa_node_to_heap_map(int nheaps)
5220 { // called right after GCHeap::Init() for each heap is finished
5221 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5222 // with 0s during initialization, and will be treated as one node
5223 numa_node_to_heap_map[0] = 0;
5226 for (int i=1; i < nheaps; i++)
5228 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5229 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5231 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5234 static void get_heap_range_for_heap(int hn, int* start, int* end)
5235 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5236 // and treated as in one node. thus: start=0, end=n_heaps
5237 uint16_t numa_node = heap_no_to_numa_node[hn];
5238 *start = (int)numa_node_to_heap_map[numa_node];
5239 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5242 uint8_t* heap_select::sniff_buffer;
5243 unsigned heap_select::n_sniff_buffers;
5244 unsigned heap_select::cur_sniff_index;
5245 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5246 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5247 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5248 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5249 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5250 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5252 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5255 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5259 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5263 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5274 destroy_thread_support();
5280 void gc_heap::destroy_thread_support ()
5282 if (ee_suspend_event.IsValid())
5284 ee_suspend_event.CloseEvent();
5286 if (gc_start_event.IsValid())
5288 gc_start_event.CloseEvent();
5292 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5294 affinity->Group = GCThreadAffinity::None;
5295 affinity->Processor = GCThreadAffinity::None;
5298 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5301 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5303 if (bit_number == gpn)
5305 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5306 affinity->Processor = gpn;
5307 affinity->Group = gn;
5308 heap_select::set_cpu_group_for_heap(heap_number, gn);
5309 heap_select::set_group_proc_for_heap(heap_number, gpn);
5310 if (GCToOSInterface::CanEnableGCNumaAware())
5312 PROCESSOR_NUMBER proc_no;
5314 proc_no.Number = (uint8_t)gpn;
5315 proc_no.Reserved = 0;
5317 uint16_t node_no = 0;
5318 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5319 heap_select::set_numa_node_for_heap(heap_number, node_no);
5322 { // no numa setting, each cpu group is treated as a node
5323 heap_select::set_numa_node_for_heap(heap_number, gn);
5331 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5333 affinity->Group = GCThreadAffinity::None;
5334 affinity->Processor = GCThreadAffinity::None;
5336 uintptr_t pmask = process_mask;
5338 uint8_t proc_number = 0;
5339 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5341 if ((mask & pmask) != 0)
5343 if (bit_number == heap_number)
5345 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5346 affinity->Processor = proc_number;
5347 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5348 if (GCToOSInterface::CanEnableGCNumaAware())
5350 uint16_t node_no = 0;
5351 PROCESSOR_NUMBER proc_no;
5353 proc_no.Number = (uint8_t)proc_number;
5354 proc_no.Reserved = 0;
5355 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5357 heap_select::set_numa_node_for_heap(heap_number, node_no);
5368 bool gc_heap::create_gc_thread ()
5370 dprintf (3, ("Creating gc thread\n"));
5371 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5375 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5377 void gc_heap::gc_thread_function ()
5379 assert (gc_done_event.IsValid());
5380 assert (gc_start_event.IsValid());
5381 dprintf (3, ("gc thread started"));
5383 heap_select::init_cpu_mapping(this, heap_number);
5387 assert (!gc_t_join.joined());
5389 if (heap_number == 0)
5391 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5393 BEGIN_TIMING(suspend_ee_during_log);
5394 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5395 END_TIMING(suspend_ee_during_log);
5397 proceed_with_gc_p = TRUE;
5399 if (!should_proceed_with_gc())
5401 update_collection_counts_for_no_gc();
5402 proceed_with_gc_p = FALSE;
5406 settings.init_mechanisms();
5407 gc_start_event.Set();
5409 dprintf (3, ("%d gc thread waiting...", heap_number));
5413 gc_start_event.Wait(INFINITE, FALSE);
5414 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5417 assert ((heap_number == 0) || proceed_with_gc_p);
5419 if (proceed_with_gc_p)
5421 garbage_collect (GCHeap::GcCondemnedGeneration);
5423 if (pm_trigger_full_gc)
5425 garbage_collect_pm_full_gc();
5429 if (heap_number == 0)
5431 if (proceed_with_gc_p && (!settings.concurrent))
5436 #ifdef BACKGROUND_GC
5437 recover_bgc_settings();
5438 #endif //BACKGROUND_GC
5440 #ifdef MULTIPLE_HEAPS
5441 for (int i = 0; i < gc_heap::n_heaps; i++)
5443 gc_heap* hp = gc_heap::g_heaps[i];
5444 hp->add_saved_spinlock_info (false, me_release, mt_block_gc);
5445 leave_spin_lock(&hp->more_space_lock_soh);
5447 #endif //MULTIPLE_HEAPS
5449 gc_heap::gc_started = FALSE;
5451 BEGIN_TIMING(restart_ee_during_log);
5452 GCToEEInterface::RestartEE(TRUE);
5453 END_TIMING(restart_ee_during_log);
5454 process_sync_log_stats();
5456 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5457 leave_spin_lock (&gc_heap::gc_lock);
5459 gc_heap::internal_gc_done = true;
5461 if (proceed_with_gc_p)
5465 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5466 // we still need to set the gc_done_event for those threads.
5467 for (int i = 0; i < gc_heap::n_heaps; i++)
5469 gc_heap* hp = gc_heap::g_heaps[i];
5476 int spin_count = 32 * (gc_heap::n_heaps - 1);
5478 // wait until RestartEE has progressed to a stage where we can restart user threads
5479 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5481 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5488 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5491 #endif //MULTIPLE_HEAPS
5493 bool gc_heap::virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number)
5495 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5496 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5497 // a host. This will need to be added later.
5498 #if !defined(FEATURE_CORECLR) && !defined(BUILD_AS_STANDALONE)
5499 if (!CLRMemoryHosted())
5502 if (GCToOSInterface::CanEnableGCNumaAware())
5504 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5505 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5510 UNREFERENCED_PARAMETER(h_number);
5513 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5514 return GCToOSInterface::VirtualCommit(addr, size);
5517 bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* hard_limit_exceeded_p)
5520 assert (heap_hard_limit == 0);
5523 if (heap_hard_limit)
5525 bool exceeded_p = false;
5527 check_commit_cs.Enter();
5529 if ((current_total_committed + size) > heap_hard_limit)
5531 dprintf (1, ("%Id + %Id = %Id > limit",
5532 current_total_committed, size,
5533 (current_total_committed + size),
5540 current_total_committed += size;
5542 current_total_committed_bookkeeping += size;
5545 check_commit_cs.Leave();
5547 if (hard_limit_exceeded_p)
5548 *hard_limit_exceeded_p = exceeded_p;
5552 dprintf (1, ("can't commit %Ix for %Id bytes > HARD LIMIT %Id", (size_t)address, size, heap_hard_limit));
5557 // If it's a valid heap number it means it's commiting for memory on the GC heap.
5558 bool commit_succeeded_p = ((h_number >= 0) ?
5559 virtual_alloc_commit_for_heap (address, size, h_number) :
5560 GCToOSInterface::VirtualCommit(address, size));
5562 if (!commit_succeeded_p && heap_hard_limit)
5564 check_commit_cs.Enter();
5565 dprintf (1, ("commit failed, updating %Id to %Id",
5566 current_total_committed, (current_total_committed - size)));
5567 current_total_committed -= size;
5569 current_total_committed_bookkeeping -= size;
5571 check_commit_cs.Leave();
5574 return commit_succeeded_p;
5577 bool gc_heap::virtual_decommit (void* address, size_t size, int h_number)
5580 assert (heap_hard_limit == 0);
5583 bool decommit_succeeded_p = GCToOSInterface::VirtualDecommit (address, size);
5585 if (decommit_succeeded_p && heap_hard_limit)
5587 check_commit_cs.Enter();
5588 current_total_committed -= size;
5590 current_total_committed_bookkeeping -= size;
5591 check_commit_cs.Leave();
5594 return decommit_succeeded_p;
5597 #ifndef SEG_MAPPING_TABLE
5599 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5601 uint8_t* sadd = add;
5602 heap_segment* hs = 0;
5603 heap_segment* hs1 = 0;
5604 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5609 //repeat in case there is a concurrent insertion in the table.
5614 seg_table->lookup (sadd);
5615 hs1 = (heap_segment*)sadd;
5616 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5621 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5625 #endif //SEG_MAPPING_TABLE
5633 // If we want to save space we can have a pool of plug_and_gap's instead of
5634 // always having 2 allocated for each pinned plug.
5635 gap_reloc_pair saved_pre_plug;
5636 // If we decide to not compact, we need to restore the original values.
5637 gap_reloc_pair saved_pre_plug_reloc;
5639 gap_reloc_pair saved_post_plug;
5641 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5642 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5644 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5645 gap_reloc_pair saved_post_plug_reloc;
5647 // We need to calculate this after we are done with plan phase and before compact
5648 // phase because compact phase will change the bricks so relocate_address will no
5650 uint8_t* saved_pre_plug_info_reloc_start;
5652 // We need to save this because we will have no way to calculate it, unlike the
5653 // pre plug info start which is right before this plug.
5654 uint8_t* saved_post_plug_info_start;
5657 uint8_t* allocation_context_start_region;
5658 #endif //SHORT_PLUGS
5660 // How the bits in these bytes are organized:
5662 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5663 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5668 // We are seeing this is getting corrupted for a PP with a NP after.
5669 // Save it when we first set it and make sure it doesn't change.
5670 gap_reloc_pair saved_post_plug_debug;
5673 size_t get_max_short_bits()
5675 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5679 size_t get_pre_short_start_bit ()
5681 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5686 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5689 void set_pre_short()
5691 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5694 void set_pre_short_bit (size_t bit)
5696 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5699 BOOL pre_short_bit_p (size_t bit)
5701 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5704 #ifdef COLLECTIBLE_CLASS
5705 void set_pre_short_collectible()
5710 BOOL pre_short_collectible_p()
5712 return (saved_pre_p & 2);
5714 #endif //COLLECTIBLE_CLASS
5717 size_t get_post_short_start_bit ()
5719 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5724 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5727 void set_post_short()
5729 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5732 void set_post_short_bit (size_t bit)
5734 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5737 BOOL post_short_bit_p (size_t bit)
5739 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5742 #ifdef COLLECTIBLE_CLASS
5743 void set_post_short_collectible()
5748 BOOL post_short_collectible_p()
5750 return (saved_post_p & 2);
5752 #endif //COLLECTIBLE_CLASS
5754 uint8_t* get_plug_address() { return first; }
5756 BOOL has_pre_plug_info() { return saved_pre_p; }
5757 BOOL has_post_plug_info() { return saved_post_p; }
5759 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5760 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5761 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5762 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5764 // We need to temporarily recover the shortened plugs for compact phase so we can
5765 // copy over the whole plug and their related info (mark bits/cards). But we will
5766 // need to set the artificial gap back so compact phase can keep reading the plug info.
5767 // We also need to recover the saved info because we'll need to recover it later.
5769 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5770 // it again to recover the artificial gap.
5771 void swap_pre_plug_and_saved()
5773 gap_reloc_pair temp;
5774 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5775 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5776 saved_pre_plug_reloc = temp;
5779 void swap_post_plug_and_saved()
5781 gap_reloc_pair temp;
5782 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5783 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5784 saved_post_plug_reloc = temp;
5787 void swap_pre_plug_and_saved_for_profiler()
5789 gap_reloc_pair temp;
5790 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5791 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5792 saved_pre_plug = temp;
5795 void swap_post_plug_and_saved_for_profiler()
5797 gap_reloc_pair temp;
5798 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5799 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5800 saved_post_plug = temp;
5803 // We should think about whether it's really necessary to have to copy back the pre plug
5804 // info since it was already copied during compacting plugs. But if a plug doesn't move
5805 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5806 void recover_plug_info()
5810 if (gc_heap::settings.compaction)
5812 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5814 &saved_pre_plug_reloc,
5815 saved_pre_plug_info_reloc_start));
5816 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5820 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5823 (first - sizeof (plug_and_gap))));
5824 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5830 if (gc_heap::settings.compaction)
5832 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5834 &saved_post_plug_reloc,
5835 saved_post_plug_info_start));
5836 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5840 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5843 saved_post_plug_info_start));
5844 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5851 void gc_mechanisms::init_mechanisms()
5853 condemned_generation = 0;
5854 promotion = FALSE;//TRUE;
5856 #ifdef FEATURE_LOH_COMPACTION
5857 loh_compaction = gc_heap::should_compact_loh();
5859 loh_compaction = FALSE;
5860 #endif //FEATURE_LOH_COMPACTION
5861 heap_expansion = FALSE;
5864 elevation_reduced = FALSE;
5865 found_finalizers = FALSE;
5866 #ifdef BACKGROUND_GC
5867 background_p = recursive_gc_sync::background_running_p() != FALSE;
5868 allocations_allowed = TRUE;
5869 #endif //BACKGROUND_GC
5871 entry_memory_load = 0;
5872 exit_memory_load = 0;
5875 stress_induced = FALSE;
5876 #endif // STRESS_HEAP
5879 void gc_mechanisms::first_init()
5882 gen0_reduction_count = 0;
5883 should_lock_elevation = FALSE;
5884 elevation_locked_count = 0;
5885 reason = reason_empty;
5886 #ifdef BACKGROUND_GC
5887 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5889 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5890 if (debug_pause_mode >= 0)
5892 assert (debug_pause_mode <= pause_sustained_low_latency);
5893 pause_mode = (gc_pause_mode)debug_pause_mode;
5896 #else //BACKGROUND_GC
5897 pause_mode = pause_batch;
5898 #endif //BACKGROUND_GC
5903 void gc_mechanisms::record (gc_history_global* history)
5905 #ifdef MULTIPLE_HEAPS
5906 history->num_heaps = gc_heap::n_heaps;
5908 history->num_heaps = 1;
5909 #endif //MULTIPLE_HEAPS
5911 history->condemned_generation = condemned_generation;
5912 history->gen0_reduction_count = gen0_reduction_count;
5913 history->reason = reason;
5914 history->pause_mode = (int)pause_mode;
5915 history->mem_pressure = entry_memory_load;
5916 history->global_mechanims_p = 0;
5918 // start setting the boolean values.
5920 history->set_mechanism_p (global_concurrent);
5923 history->set_mechanism_p (global_compaction);
5926 history->set_mechanism_p (global_promotion);
5929 history->set_mechanism_p (global_demotion);
5932 history->set_mechanism_p (global_card_bundles);
5934 if (elevation_reduced)
5935 history->set_mechanism_p (global_elevation);
5938 /**********************************
5939 called at the beginning of GC to fix the allocated size to
5940 what is really allocated, or to turn the free area into an unused object
5941 It needs to be called after all of the other allocation contexts have been
5942 fixed since it relies on alloc_allocated.
5943 ********************************/
5945 //for_gc_p indicates that the work is being done for GC,
5946 //as opposed to concurrent heap verification
5947 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5949 UNREFERENCED_PARAMETER(for_gc_p);
5951 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5952 // still used in the allocation path during GCs.
5953 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5954 assert (generation_allocation_limit (youngest_generation) == nullptr);
5955 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5958 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5960 UNREFERENCED_PARAMETER(for_gc_p);
5963 alloc_context* acontext =
5965 generation_alloc_context (large_object_generation);
5966 assert (acontext->alloc_ptr == 0);
5967 assert (acontext->alloc_limit == 0);
5969 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5970 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5971 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5974 acontext->alloc_ptr = 0;
5975 acontext->alloc_limit = acontext->alloc_ptr;
5980 //for_gc_p indicates that the work is being done for GC,
5981 //as opposed to concurrent heap verification
5982 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5985 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5987 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5989 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5992 uint8_t* point = acontext->alloc_ptr;
5995 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5996 // the allocation area was from the free list
5997 // it was shortened by Align (min_obj_size) to make room for
5998 // at least the shortest unused object
5999 size += Align (min_obj_size, align_const);
6000 assert ((size >= Align (min_obj_size)));
6002 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
6003 (size_t)point + size ));
6004 make_unused_array (point, size);
6008 generation_free_obj_space (generation_of (0)) += size;
6009 alloc_contexts_used ++;
6015 alloc_allocated = acontext->alloc_ptr;
6016 assert (heap_segment_allocated (ephemeral_heap_segment) <=
6017 heap_segment_committed (ephemeral_heap_segment));
6018 alloc_contexts_used ++;
6023 // We need to update the alloc_bytes to reflect the portion that we have not used
6024 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
6025 acontext->alloc_ptr = 0;
6026 acontext->alloc_limit = acontext->alloc_ptr;
6030 //used by the heap verification for concurrent gc.
6031 //it nulls out the words set by fix_allocation_context for heap_verification
6032 void repair_allocation (gc_alloc_context* acontext, void*)
6034 uint8_t* point = acontext->alloc_ptr;
6038 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6039 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6040 memclr (acontext->alloc_ptr - plug_skew,
6041 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
6045 void void_allocation (gc_alloc_context* acontext, void*)
6047 uint8_t* point = acontext->alloc_ptr;
6051 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6052 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6053 acontext->alloc_ptr = 0;
6054 acontext->alloc_limit = acontext->alloc_ptr;
6058 void gc_heap::repair_allocation_contexts (BOOL repair_p)
6060 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
6063 struct fix_alloc_context_args
6069 void fix_alloc_context (gc_alloc_context* acontext, void* param)
6071 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
6072 g_theGCHeap->FixAllocContext(acontext, (void*)(size_t)(args->for_gc_p), args->heap);
6075 void gc_heap::fix_allocation_contexts (BOOL for_gc_p)
6077 fix_alloc_context_args args;
6078 args.for_gc_p = for_gc_p;
6081 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
6082 fix_youngest_allocation_area(for_gc_p);
6083 fix_large_allocation_area(for_gc_p);
6086 void gc_heap::fix_older_allocation_area (generation* older_gen)
6088 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
6089 if (generation_allocation_limit (older_gen) !=
6090 heap_segment_plan_allocated (older_gen_seg))
6092 uint8_t* point = generation_allocation_pointer (older_gen);
6094 size_t size = (generation_allocation_limit (older_gen) -
6095 generation_allocation_pointer (older_gen));
6098 assert ((size >= Align (min_obj_size)));
6099 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
6100 make_unused_array (point, size);
6101 if (size >= min_free_list)
6103 generation_allocator (older_gen)->thread_item_front (point, size);
6104 add_gen_free (older_gen->gen_num, size);
6105 generation_free_list_space (older_gen) += size;
6109 generation_free_obj_space (older_gen) += size;
6115 assert (older_gen_seg != ephemeral_heap_segment);
6116 heap_segment_plan_allocated (older_gen_seg) =
6117 generation_allocation_pointer (older_gen);
6118 generation_allocation_limit (older_gen) =
6119 generation_allocation_pointer (older_gen);
6122 generation_allocation_pointer (older_gen) = 0;
6123 generation_allocation_limit (older_gen) = 0;
6126 void gc_heap::set_allocation_heap_segment (generation* gen)
6128 uint8_t* p = generation_allocation_start (gen);
6130 heap_segment* seg = generation_allocation_segment (gen);
6131 if (in_range_for_segment (p, seg))
6134 // try ephemeral heap segment in case of heap expansion
6135 seg = ephemeral_heap_segment;
6136 if (!in_range_for_segment (p, seg))
6138 seg = heap_segment_rw (generation_start_segment (gen));
6140 PREFIX_ASSUME(seg != NULL);
6142 while (!in_range_for_segment (p, seg))
6144 seg = heap_segment_next_rw (seg);
6145 PREFIX_ASSUME(seg != NULL);
6149 generation_allocation_segment (gen) = seg;
6152 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6155 assert (Align ((size_t)start) == (size_t)start);
6156 generation_allocation_start (gen) = start;
6157 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6158 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6159 set_allocation_heap_segment (gen);
6162 #ifdef BACKGROUND_GC
6163 //TODO BACKGROUND_GC this is for test only
6165 gc_heap::disallow_new_allocation (int gen_number)
6167 UNREFERENCED_PARAMETER(gen_number);
6168 settings.allocations_allowed = FALSE;
6171 gc_heap::allow_new_allocation (int gen_number)
6173 UNREFERENCED_PARAMETER(gen_number);
6174 settings.allocations_allowed = TRUE;
6177 #endif //BACKGROUND_GC
6179 bool gc_heap::new_allocation_allowed (int gen_number)
6181 #ifdef BACKGROUND_GC
6182 //TODO BACKGROUND_GC this is for test only
6183 if (!settings.allocations_allowed)
6185 dprintf (2, ("new allocation not allowed"));
6188 #endif //BACKGROUND_GC
6190 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6192 if (gen_number != 0)
6194 // For LOH we will give it more budget before we try a GC.
6195 if (settings.concurrent)
6197 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6199 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6207 #ifndef MULTIPLE_HEAPS
6208 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6210 dprintf (3, ("evaluating allocation rate"));
6211 dynamic_data* dd0 = dynamic_data_of (0);
6212 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6215 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6216 if ((ctime - allocation_running_time) > 1000)
6218 dprintf (2, (">1s since last gen0 gc"));
6223 allocation_running_amount = dd_new_allocation (dd0);
6227 #endif //MULTIPLE_HEAPS
6232 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6234 return dd_desired_allocation (dynamic_data_of (gen_number));
6238 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6240 return dd_new_allocation (dynamic_data_of (gen_number));
6243 //return the amount allocated so far in gen_number
6245 ptrdiff_t gc_heap::get_allocation (int gen_number)
6247 dynamic_data* dd = dynamic_data_of (gen_number);
6249 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6253 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6255 size_t new_size = max (init_len, 2*len);
6256 mark* tmp = new (nothrow) mark [new_size];
6259 memcpy (tmp, m, len * sizeof (mark));
6267 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6273 uint8_t* pinned_plug (mark* m)
6279 size_t& pinned_len (mark* m)
6285 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6287 m->len = pinned_plug (m) - pin_free_space_start;
6289 m->allocation_context_start_region = pin_free_space_start;
6290 #endif //SHORT_PLUGS
6295 uint8_t*& pin_allocation_context_start_region (mark* m)
6297 return m->allocation_context_start_region;
6300 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6302 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6303 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6304 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6305 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6306 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6307 return plug_start_in_saved;
6311 void set_padding_in_expand (uint8_t* old_loc,
6312 BOOL set_padding_on_saved_p,
6313 mark* pinned_plug_entry)
6315 if (set_padding_on_saved_p)
6317 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6321 set_plug_padded (old_loc);
6326 void clear_padding_in_expand (uint8_t* old_loc,
6327 BOOL set_padding_on_saved_p,
6328 mark* pinned_plug_entry)
6330 if (set_padding_on_saved_p)
6332 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6336 clear_plug_padded (old_loc);
6339 #endif //SHORT_PLUGS
6341 void gc_heap::reset_pinned_queue()
6347 void gc_heap::reset_pinned_queue_bos()
6352 // last_pinned_plug is only for asserting purpose.
6353 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6355 if (last_pinned_plug)
6357 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6358 assert (last_pinned_plug == last_m.first);
6359 if (last_m.saved_post_p)
6361 last_m.saved_post_p = FALSE;
6362 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6363 // We need to recover what the gap has overwritten.
6364 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6366 last_m.len += plug_size;
6367 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6371 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6373 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6374 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6375 if (!(pinned_plug_que_empty_p()))
6377 mark* oldest_entry = oldest_pin();
6378 uint8_t* plug = pinned_plug (oldest_entry);
6379 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6381 alloc_limit = pinned_plug (oldest_entry);
6382 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6383 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6388 void gc_heap::set_allocator_next_pin (generation* gen)
6390 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6391 if (!(pinned_plug_que_empty_p()))
6393 mark* oldest_entry = oldest_pin();
6394 uint8_t* plug = pinned_plug (oldest_entry);
6395 if ((plug >= generation_allocation_pointer (gen)) &&
6396 (plug < generation_allocation_limit (gen)))
6398 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6399 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6401 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6402 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6405 assert (!((plug < generation_allocation_pointer (gen)) &&
6406 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6410 // After we set the info, we increase tos.
6411 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6413 UNREFERENCED_PARAMETER(last_pinned_plug);
6415 mark& m = mark_stack_array[mark_stack_tos];
6416 assert (m.first == last_pinned_plug);
6420 set_allocator_next_pin (alloc_pointer, alloc_limit);
6423 // After we set the info, we increase tos.
6424 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6426 UNREFERENCED_PARAMETER(last_pinned_plug);
6428 mark& m = mark_stack_array[mark_stack_tos];
6429 assert (m.first == last_pinned_plug);
6434 // Why are we checking here? gen is never 0.
6437 set_allocator_next_pin (gen);
6441 size_t gc_heap::deque_pinned_plug ()
6443 dprintf (3, ("dequed: %Id", mark_stack_bos));
6444 size_t m = mark_stack_bos;
6450 mark* gc_heap::pinned_plug_of (size_t bos)
6452 return &mark_stack_array [ bos ];
6456 mark* gc_heap::oldest_pin ()
6458 return pinned_plug_of (mark_stack_bos);
6462 BOOL gc_heap::pinned_plug_que_empty_p ()
6464 return (mark_stack_bos == mark_stack_tos);
6468 mark* gc_heap::before_oldest_pin()
6470 if (mark_stack_bos >= 1)
6471 return pinned_plug_of (mark_stack_bos-1);
6477 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6479 return ((o >= ephemeral_low) && (o < ephemeral_high));
6484 int& gc_heap::mark_stack_busy()
6486 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6490 void gc_heap::make_mark_stack (mark* arr)
6492 reset_pinned_queue();
6493 mark_stack_array = arr;
6494 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6496 mark_stack_busy() = 0;
6500 #ifdef BACKGROUND_GC
6502 size_t& gc_heap::bpromoted_bytes(int thread)
6504 #ifdef MULTIPLE_HEAPS
6505 return g_bpromoted [thread*16];
6506 #else //MULTIPLE_HEAPS
6507 UNREFERENCED_PARAMETER(thread);
6509 #endif //MULTIPLE_HEAPS
6512 void gc_heap::make_background_mark_stack (uint8_t** arr)
6514 background_mark_stack_array = arr;
6515 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6516 background_mark_stack_tos = arr;
6519 void gc_heap::make_c_mark_list (uint8_t** arr)
6522 c_mark_list_index = 0;
6523 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6525 #endif //BACKGROUND_GC
6530 // The card bundle keeps track of groups of card words.
6531 static const size_t card_bundle_word_width = 32;
6533 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6534 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6537 size_t card_bundle_word (size_t cardb)
6539 return cardb / card_bundle_word_width;
6543 uint32_t card_bundle_bit (size_t cardb)
6545 return (uint32_t)(cardb % card_bundle_word_width);
6548 size_t align_cardw_on_bundle (size_t cardw)
6550 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6553 // Get the card bundle representing a card word
6554 size_t cardw_card_bundle (size_t cardw)
6556 return cardw / card_bundle_size;
6559 // Get the first card word in a card bundle
6560 size_t card_bundle_cardw (size_t cardb)
6562 return cardb * card_bundle_size;
6565 // Clear the specified card bundle
6566 void gc_heap::card_bundle_clear (size_t cardb)
6568 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6569 dprintf (2, ("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6570 (size_t)card_bundle_cardw (cardb+1)));
6573 void gc_heap::card_bundle_set (size_t cardb)
6575 if (!card_bundle_set_p (cardb))
6577 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6581 // Set the card bundle bits between start_cardb and end_cardb
6582 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6584 if (start_cardb == end_cardb)
6586 card_bundle_set(start_cardb);
6590 size_t start_word = card_bundle_word (start_cardb);
6591 size_t end_word = card_bundle_word (end_cardb);
6593 if (start_word < end_word)
6595 // Set the partial words
6596 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6598 if (card_bundle_bit (end_cardb))
6599 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6601 // Set the full words
6602 for (size_t i = start_word + 1; i < end_word; i++)
6603 card_bundle_table [i] = ~0u;
6607 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6608 lowbits (~0u, card_bundle_bit (end_cardb)));
6612 // Indicates whether the specified bundle is set.
6613 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6615 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6618 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6619 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6621 // Number of heap bytes represented by a card bundle word
6622 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6624 // Align the start of the region down
6625 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6627 // Align the end of the region up
6628 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6630 // Make sure they're really aligned
6631 assert (((size_t)from & (cbw_span - 1)) == 0);
6632 assert (((size_t)end & (cbw_span - 1)) == 0);
6634 return ((end - from) / cbw_span) * sizeof (uint32_t);
6637 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6638 // where a theoretical card bundle table that represents every address (starting from 0) would
6639 // start if the bundle word representing the address were to be located at the pointer passed in.
6640 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6641 // for a given address is using a simple shift operation on the address.
6642 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6644 // The number of bytes of heap memory represented by a card bundle word
6645 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6647 // Each card bundle word is 32 bits
6648 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6651 void gc_heap::enable_card_bundles ()
6653 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6655 dprintf (1, ("Enabling card bundles"));
6657 // We initially set all of the card bundles
6658 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6659 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6660 settings.card_bundles = TRUE;
6664 BOOL gc_heap::card_bundles_enabled ()
6666 return settings.card_bundles;
6669 #endif // CARD_BUNDLE
6671 #if defined (_TARGET_AMD64_)
6672 #define brick_size ((size_t)4096)
6674 #define brick_size ((size_t)2048)
6675 #endif //_TARGET_AMD64_
6678 size_t gc_heap::brick_of (uint8_t* add)
6680 return (size_t)(add - lowest_address) / brick_size;
6684 uint8_t* gc_heap::brick_address (size_t brick)
6686 return lowest_address + (brick_size * brick);
6690 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6692 for (size_t i = brick_of (from);i < brick_of (end); i++)
6696 //codes for the brick entries:
6697 //entry == 0 -> not assigned
6698 //entry >0 offset is entry-1
6699 //entry <0 jump back entry bricks
6703 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6709 assert (val < 32767);
6711 brick_table [index] = (short)val+1;
6713 brick_table [index] = (short)val;
6717 int gc_heap::get_brick_entry (size_t index)
6719 #ifdef MULTIPLE_HEAPS
6720 return VolatileLoadWithoutBarrier(&brick_table [index]);
6722 return brick_table[index];
6728 uint8_t* align_on_brick (uint8_t* add)
6730 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6734 uint8_t* align_lower_brick (uint8_t* add)
6736 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6739 size_t size_brick_of (uint8_t* from, uint8_t* end)
6741 assert (((size_t)from & (brick_size-1)) == 0);
6742 assert (((size_t)end & (brick_size-1)) == 0);
6744 return ((end - from) / brick_size) * sizeof (short);
6748 uint8_t* gc_heap::card_address (size_t card)
6750 return (uint8_t*) (card_size * card);
6754 size_t gc_heap::card_of ( uint8_t* object)
6756 return (size_t)(object) / card_size;
6760 size_t gc_heap::card_to_brick (size_t card)
6762 return brick_of (card_address (card));
6766 uint8_t* align_on_card (uint8_t* add)
6768 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6771 uint8_t* align_on_card_word (uint8_t* add)
6773 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6777 uint8_t* align_lower_card (uint8_t* add)
6779 return (uint8_t*)((size_t)add & ~(card_size-1));
6783 void gc_heap::clear_card (size_t card)
6785 card_table [card_word (card)] =
6786 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6787 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6788 (size_t)card_address (card+1)));
6792 void gc_heap::set_card (size_t card)
6794 size_t word = card_word (card);
6795 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6797 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6798 // Also set the card bundle that corresponds to the card
6799 size_t bundle_to_set = cardw_card_bundle(word);
6801 card_bundle_set(bundle_to_set);
6803 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6804 assert(card_bundle_set_p(bundle_to_set) != 0);
6809 BOOL gc_heap::card_set_p (size_t card)
6811 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6814 // Returns the number of DWORDs in the card table that cover the
6815 // range of addresses [from, end[.
6816 size_t count_card_of (uint8_t* from, uint8_t* end)
6818 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6821 // Returns the number of bytes to allocate for a card table
6822 // that covers the range of addresses [from, end[.
6823 size_t size_card_of (uint8_t* from, uint8_t* end)
6825 return count_card_of (from, end) * sizeof(uint32_t);
6828 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6829 class card_table_info
6833 uint8_t* lowest_address;
6834 uint8_t* highest_address;
6838 uint32_t* card_bundle_table;
6839 #endif //CARD_BUNDLE
6841 // mark_array is always at the end of the data structure because we
6842 // want to be able to make one commit call for everything before it.
6844 uint32_t* mark_array;
6848 uint32_t* next_card_table;
6851 //These are accessors on untranslated cardtable
6853 unsigned& card_table_refcount (uint32_t* c_table)
6855 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6859 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6861 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6864 uint32_t* translate_card_table (uint32_t* ct)
6866 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6870 uint8_t*& card_table_highest_address (uint32_t* c_table)
6872 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6876 short*& card_table_brick_table (uint32_t* c_table)
6878 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6883 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6885 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6887 #endif //CARD_BUNDLE
6890 /* Support for mark_array */
6893 uint32_t*& card_table_mark_array (uint32_t* c_table)
6895 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6899 #define mark_bit_pitch ((size_t)16)
6901 #define mark_bit_pitch ((size_t)8)
6903 #define mark_word_width ((size_t)32)
6904 #define mark_word_size (mark_word_width * mark_bit_pitch)
6907 uint8_t* align_on_mark_bit (uint8_t* add)
6909 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6913 uint8_t* align_lower_mark_bit (uint8_t* add)
6915 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6919 BOOL is_aligned_on_mark_word (uint8_t* add)
6921 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6925 uint8_t* align_on_mark_word (uint8_t* add)
6927 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6931 uint8_t* align_lower_mark_word (uint8_t* add)
6933 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6937 size_t mark_bit_of (uint8_t* add)
6939 return ((size_t)add / mark_bit_pitch);
6943 unsigned int mark_bit_bit (size_t mark_bit)
6945 return (unsigned int)(mark_bit % mark_word_width);
6949 size_t mark_bit_word (size_t mark_bit)
6951 return (mark_bit / mark_word_width);
6955 size_t mark_word_of (uint8_t* add)
6957 return ((size_t)add) / mark_word_size;
6960 uint8_t* mark_word_address (size_t wd)
6962 return (uint8_t*)(wd*mark_word_size);
6965 uint8_t* mark_bit_address (size_t mark_bit)
6967 return (uint8_t*)(mark_bit*mark_bit_pitch);
6971 size_t mark_bit_bit_of (uint8_t* add)
6973 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6977 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6979 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6983 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6985 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6989 void gc_heap::mark_array_set_marked (uint8_t* add)
6991 size_t index = mark_word_of (add);
6992 uint32_t val = (1 << mark_bit_bit_of (add));
6993 #ifdef MULTIPLE_HEAPS
6994 Interlocked::Or (&(mark_array [index]), val);
6996 mark_array [index] |= val;
7001 void gc_heap::mark_array_clear_marked (uint8_t* add)
7003 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
7006 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
7008 assert (((size_t)from & ((mark_word_size)-1)) == 0);
7009 assert (((size_t)end & ((mark_word_size)-1)) == 0);
7010 return sizeof (uint32_t)*(((end - from) / mark_word_size));
7013 //In order to eliminate the lowest_address in the mark array
7014 //computations (mark_word_of, etc) mark_array is offset
7015 // according to the lowest_address.
7016 uint32_t* translate_mark_array (uint32_t* ma)
7018 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
7021 // from and end must be page aligned addresses.
7022 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
7023 #ifdef FEATURE_BASICFREEZE
7024 , BOOL read_only/*=FALSE*/
7025 #endif // FEATURE_BASICFREEZE
7028 if(!gc_can_use_concurrent)
7031 #ifdef FEATURE_BASICFREEZE
7033 #endif // FEATURE_BASICFREEZE
7035 assert (from == align_on_mark_word (from));
7037 assert (end == align_on_mark_word (end));
7039 #ifdef BACKGROUND_GC
7040 uint8_t* current_lowest_address = background_saved_lowest_address;
7041 uint8_t* current_highest_address = background_saved_highest_address;
7043 uint8_t* current_lowest_address = lowest_address;
7044 uint8_t* current_highest_address = highest_address;
7045 #endif //BACKGROUND_GC
7047 //there is a possibility of the addresses to be
7048 //outside of the covered range because of a newly allocated
7049 //large object segment
7050 if ((end <= current_highest_address) && (from >= current_lowest_address))
7052 size_t beg_word = mark_word_of (align_on_mark_word (from));
7053 MAYBE_UNUSED_VAR(beg_word);
7054 //align end word to make sure to cover the address
7055 size_t end_word = mark_word_of (align_on_mark_word (end));
7056 MAYBE_UNUSED_VAR(end_word);
7057 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
7058 (size_t)mark_word_address (beg_word),
7059 (size_t)mark_word_address (end_word),
7060 (size_t)from, (size_t)end,
7061 (check_only ? "check_only" : "clear")));
7065 while (op < mark_word_address (beg_word))
7067 mark_array_clear_marked (op);
7068 op += mark_bit_pitch;
7071 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
7076 //Beware, it is assumed that the mark array word straddling
7077 //start has been cleared before
7078 //verify that the array is empty.
7079 size_t markw = mark_word_of (align_on_mark_word (from));
7080 size_t markw_end = mark_word_of (align_on_mark_word (end));
7081 while (markw < markw_end)
7083 assert (!(mark_array [markw]));
7086 uint8_t* p = mark_word_address (markw_end);
7089 assert (!(mark_array_marked (p)));
7098 //These work on untranslated card tables
7100 uint32_t*& card_table_next (uint32_t* c_table)
7102 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
7106 size_t& card_table_size (uint32_t* c_table)
7108 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
7111 void own_card_table (uint32_t* c_table)
7113 card_table_refcount (c_table) += 1;
7116 void destroy_card_table (uint32_t* c_table);
7118 void delete_next_card_table (uint32_t* c_table)
7120 uint32_t* n_table = card_table_next (c_table);
7123 if (card_table_next (n_table))
7125 delete_next_card_table (n_table);
7127 if (card_table_refcount (n_table) == 0)
7129 destroy_card_table (n_table);
7130 card_table_next (c_table) = 0;
7135 void release_card_table (uint32_t* c_table)
7137 assert (card_table_refcount (c_table) >0);
7138 card_table_refcount (c_table) -= 1;
7139 if (card_table_refcount (c_table) == 0)
7141 delete_next_card_table (c_table);
7142 if (card_table_next (c_table) == 0)
7144 destroy_card_table (c_table);
7145 // sever the link from the parent
7146 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7148 g_gc_card_table = 0;
7150 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7151 g_gc_card_bundle_table = 0;
7153 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7154 SoftwareWriteWatch::StaticClose();
7155 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7159 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7162 while (p_table && (card_table_next (p_table) != c_table))
7163 p_table = card_table_next (p_table);
7164 card_table_next (p_table) = 0;
7171 void destroy_card_table (uint32_t* c_table)
7173 // delete (uint32_t*)&card_table_refcount(c_table);
7175 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7176 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7179 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7181 assert (g_gc_lowest_address == start);
7182 assert (g_gc_highest_address == end);
7184 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7186 size_t bs = size_brick_of (start, end);
7187 size_t cs = size_card_of (start, end);
7189 size_t ms = (gc_can_use_concurrent ?
7190 size_mark_array_of (start, end) :
7199 if (can_use_write_watch_for_card_table())
7201 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7202 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7203 // If we're not manually managing the card bundles, we will need to use OS write
7204 // watch APIs over this region to track changes.
7205 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7208 #endif //CARD_BUNDLE
7211 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7212 size_t sw_ww_table_offset = 0;
7213 if (gc_can_use_concurrent)
7215 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7216 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7217 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7219 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7221 #ifdef GROWABLE_SEG_MAPPING_TABLE
7222 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7223 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7224 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7226 st += (st_table_offset_aligned - st_table_offset);
7227 #else //GROWABLE_SEG_MAPPING_TABLE
7229 #endif //GROWABLE_SEG_MAPPING_TABLE
7231 // it is impossible for alloc_size to overflow due bounds on each of
7233 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7234 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7239 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7240 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7242 // mark array will be committed separately (per segment).
7243 size_t commit_size = alloc_size - ms;
7245 if (!virtual_commit (mem, commit_size))
7247 dprintf (1, ("Card table commit failed"));
7248 GCToOSInterface::VirtualRelease (mem, alloc_size);
7252 // initialize the ref count
7253 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7254 card_table_refcount (ct) = 0;
7255 card_table_lowest_address (ct) = start;
7256 card_table_highest_address (ct) = end;
7257 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7258 card_table_size (ct) = alloc_size;
7259 card_table_next (ct) = 0;
7262 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7264 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7265 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7268 #endif //CARD_BUNDLE
7270 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7271 if (gc_can_use_concurrent)
7273 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7275 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7277 #ifdef GROWABLE_SEG_MAPPING_TABLE
7278 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7279 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7280 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7281 #endif //GROWABLE_SEG_MAPPING_TABLE
7284 if (gc_can_use_concurrent)
7285 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7287 card_table_mark_array (ct) = NULL;
7290 return translate_card_table(ct);
7293 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7295 #ifdef MULTIPLE_HEAPS
7296 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7298 gc_heap* hp = gc_heap::g_heaps [hn];
7299 hp->fgm_result.set_fgm (f, s, loh_p);
7301 #else //MULTIPLE_HEAPS
7302 fgm_result.set_fgm (f, s, loh_p);
7303 #endif //MULTIPLE_HEAPS
7306 //returns 0 for success, -1 otherwise
7307 // We are doing all the decommitting here because we want to make sure we have
7308 // enough memory to do so - if we do this during copy_brick_card_table and
7309 // and fail to decommit it would make the failure case very complicated to
7310 // handle. This way we can waste some decommit if we call this multiple
7311 // times before the next FGC but it's easier to handle the failure case.
7312 int gc_heap::grow_brick_card_tables (uint8_t* start,
7315 heap_segment* new_seg,
7319 uint8_t* la = g_gc_lowest_address;
7320 uint8_t* ha = g_gc_highest_address;
7321 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7322 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7323 seg_mapping* new_seg_mapping_table = nullptr;
7324 #ifdef BACKGROUND_GC
7325 // This value is only for logging purpose - it's not necessarily exactly what we
7326 // would commit for mark array but close enough for diagnostics purpose.
7327 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7328 #endif //BACKGROUND_GC
7330 // See if the address is already covered
7331 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7334 //modify the higest address so the span covered
7335 //is twice the previous one.
7336 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7337 // On non-Windows systems, we get only an approximate value that can possibly be
7338 // slightly lower than the saved_g_highest_address.
7339 // In such case, we set the top to the saved_g_highest_address so that the
7340 // card and brick tables always cover the whole new range.
7341 if (top < saved_g_highest_address)
7343 top = saved_g_highest_address;
7347 if (ps > (uint64_t)200*1024*1024*1024)
7348 ps += (uint64_t)100*1024*1024*1024;
7353 if (saved_g_lowest_address < g_gc_lowest_address)
7355 if (ps > (size_t)g_gc_lowest_address)
7356 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7359 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7360 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7364 if (saved_g_highest_address > g_gc_highest_address)
7366 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7367 if (saved_g_highest_address > top)
7368 saved_g_highest_address = top;
7371 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7372 (size_t)saved_g_lowest_address,
7373 (size_t)saved_g_highest_address));
7375 bool write_barrier_updated = false;
7376 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7377 uint32_t* saved_g_card_table = g_gc_card_table;
7379 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7380 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7384 uint32_t* translated_ct = 0;
7387 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7388 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7391 size_t ms = (gc_heap::gc_can_use_concurrent ?
7392 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7401 if (can_use_write_watch_for_card_table())
7403 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7405 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7406 // If we're not manually managing the card bundles, we will need to use OS write
7407 // watch APIs over this region to track changes.
7408 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7411 #endif //CARD_BUNDLE
7414 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7415 size_t sw_ww_table_offset = 0;
7416 if (gc_can_use_concurrent)
7418 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7419 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7421 sw_ww_table_offset -
7422 sw_ww_size_before_table +
7423 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7425 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7427 #ifdef GROWABLE_SEG_MAPPING_TABLE
7428 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7429 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7430 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7431 st += (st_table_offset_aligned - st_table_offset);
7432 #else //GROWABLE_SEG_MAPPING_TABLE
7434 #endif //GROWABLE_SEG_MAPPING_TABLE
7436 // it is impossible for alloc_size to overflow due bounds on each of
7438 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7439 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7440 cs, bs, cb, wws, st, ms));
7442 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7446 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7450 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7451 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7454 // mark array will be committed separately (per segment).
7455 size_t commit_size = alloc_size - ms;
7457 if (!virtual_commit (mem, commit_size))
7459 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7460 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7465 ct = (uint32_t*)(mem + sizeof (card_table_info));
7466 card_table_refcount (ct) = 0;
7467 card_table_lowest_address (ct) = saved_g_lowest_address;
7468 card_table_highest_address (ct) = saved_g_highest_address;
7469 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7471 //clear the card table
7473 memclr ((uint8_t*)ct,
7474 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7475 (card_size * card_word_width))
7476 + sizeof (uint32_t)));
7479 bt = (short*)((uint8_t*)ct + cs);
7481 // No initialization needed, will be done in copy_brick_card
7483 card_table_brick_table (ct) = bt;
7486 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7487 //set all bundle to look at all of the cards
7488 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7489 #endif //CARD_BUNDLE
7491 #ifdef GROWABLE_SEG_MAPPING_TABLE
7493 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7494 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7495 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7496 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7497 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7498 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7500 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7501 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7502 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7503 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7504 // if an OOM occurs.
7506 #endif //GROWABLE_SEG_MAPPING_TABLE
7509 if(gc_can_use_concurrent)
7510 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7512 card_table_mark_array (ct) = NULL;
7515 translated_ct = translate_card_table (ct);
7517 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7518 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7520 #ifdef BACKGROUND_GC
7521 if (hp->should_commit_mark_array())
7523 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7524 saved_g_lowest_address, saved_g_highest_address,
7525 card_table_mark_array (ct),
7526 translate_mark_array (card_table_mark_array (ct))));
7527 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7528 if (!commit_new_mark_array_global (new_mark_array))
7530 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7531 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7535 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7537 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7538 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7544 clear_commit_flag_global();
7546 #endif //BACKGROUND_GC
7548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7549 if (gc_can_use_concurrent)
7551 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7552 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7553 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7554 // table info lazily as done for card tables.
7556 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7557 // from a GC thread which means we are in a blocking GC and also suspended.
7558 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7559 if (!is_runtime_suspended)
7561 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7562 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7563 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7564 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7565 // g_gc_highest_address.
7569 g_gc_card_table = translated_ct;
7571 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7572 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7575 SoftwareWriteWatch::SetResizedUntranslatedTable(
7576 mem + sw_ww_table_offset,
7577 saved_g_lowest_address,
7578 saved_g_highest_address);
7580 seg_mapping_table = new_seg_mapping_table;
7582 // Since the runtime is already suspended, update the write barrier here as well.
7583 // This passes a bool telling whether we need to switch to the post
7584 // grow version of the write barrier. This test tells us if the new
7585 // segment was allocated at a lower address than the old, requiring
7586 // that we start doing an upper bounds check in the write barrier.
7587 g_gc_lowest_address = saved_g_lowest_address;
7588 g_gc_highest_address = saved_g_highest_address;
7589 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7590 write_barrier_updated = true;
7592 if (!is_runtime_suspended)
7598 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7600 g_gc_card_table = translated_ct;
7602 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7603 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7607 if (!write_barrier_updated)
7609 seg_mapping_table = new_seg_mapping_table;
7610 GCToOSInterface::FlushProcessWriteBuffers();
7611 g_gc_lowest_address = saved_g_lowest_address;
7612 g_gc_highest_address = saved_g_highest_address;
7614 // This passes a bool telling whether we need to switch to the post
7615 // grow version of the write barrier. This test tells us if the new
7616 // segment was allocated at a lower address than the old, requiring
7617 // that we start doing an upper bounds check in the write barrier.
7618 // This will also suspend the runtime if the write barrier type needs
7619 // to be changed, so we are doing this after all global state has
7620 // been updated. See the comment above suspend_EE() above for more
7622 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7628 //cleanup mess and return -1;
7632 assert(g_gc_card_table == saved_g_card_table);
7634 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7635 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7638 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7639 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7641 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7642 assert (!"release failed");
7650 #ifdef BACKGROUND_GC
7651 if (hp->should_commit_mark_array())
7653 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7654 if (!commit_mark_array_new_seg (hp, new_seg))
7656 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7657 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7661 #endif //BACKGROUND_GC
7667 //copy all of the arrays managed by the card table for a page aligned range
7668 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7669 short* old_brick_table,
7671 uint8_t* start, uint8_t* end)
7673 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7676 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7679 short* brick_start = &brick_table [brick_of (start)];
7680 if (old_brick_table)
7682 // segments are always on page boundaries
7683 memcpy (brick_start, &old_brick_table[brick_offset],
7684 size_brick_of (start, end));
7689 // This is a large heap, just clear the brick table
7692 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7694 #ifdef BACKGROUND_GC
7695 UNREFERENCED_PARAMETER(seg);
7696 if (recursive_gc_sync::background_running_p())
7698 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7700 // We don't need to go through all the card tables here because
7701 // we only need to copy from the GC version of the mark array - when we
7702 // mark (even in allocate_large_object) we always use that mark array.
7703 if ((card_table_highest_address (old_ct) >= start) &&
7704 (card_table_lowest_address (old_ct) <= end))
7706 if ((background_saved_highest_address >= start) &&
7707 (background_saved_lowest_address <= end))
7709 //copy the mark bits
7710 // segments are always on page boundaries
7711 uint8_t* m_start = max (background_saved_lowest_address, start);
7712 uint8_t* m_end = min (background_saved_highest_address, end);
7713 memcpy (&mark_array[mark_word_of (m_start)],
7714 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7715 size_mark_array_of (m_start, m_end));
7720 //only large segments can be out of range
7721 assert (old_brick_table == 0);
7724 #else //BACKGROUND_GC
7726 clear_mark_array (start, heap_segment_committed(seg));
7727 #endif //BACKGROUND_GC
7730 // n way merge with all of the card table ever used in between
7731 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7734 while (card_table_next (old_ct) != ct)
7736 //copy if old card table contained [start, end[
7737 if ((card_table_highest_address (ct) >= end) &&
7738 (card_table_lowest_address (ct) <= start))
7740 // or the card_tables
7742 size_t start_word = card_word (card_of (start));
7744 uint32_t* dest = &card_table[start_word];
7745 uint32_t* src = &((translate_card_table (ct))[start_word]);
7746 ptrdiff_t count = count_card_of (start, end);
7747 for (int x = 0; x < count; x++)
7751 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7754 card_bundle_set(cardw_card_bundle(start_word+x));
7762 ct = card_table_next (ct);
7766 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7767 void gc_heap::init_brick_card_range (heap_segment* seg)
7769 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7770 (size_t)heap_segment_mem (seg),
7771 (size_t)heap_segment_allocated (seg)));
7773 // initialize the brick table
7774 for (size_t b = brick_of (heap_segment_mem (seg));
7775 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7782 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7785 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7789 clear_card_for_addresses (heap_segment_mem (seg),
7790 heap_segment_allocated (seg));
7793 void gc_heap::copy_brick_card_table()
7795 uint8_t* la = lowest_address;
7796 uint8_t* ha = highest_address;
7797 MAYBE_UNUSED_VAR(ha);
7798 uint32_t* old_card_table = card_table;
7799 short* old_brick_table = brick_table;
7801 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7802 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7804 /* todo: Need a global lock for this */
7805 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7806 own_card_table (ct);
7807 card_table = translate_card_table (ct);
7808 /* End of global lock */
7809 highest_address = card_table_highest_address (ct);
7810 lowest_address = card_table_lowest_address (ct);
7812 brick_table = card_table_brick_table (ct);
7815 if (gc_can_use_concurrent)
7817 mark_array = translate_mark_array (card_table_mark_array (ct));
7818 assert (mark_word_of (g_gc_highest_address) ==
7819 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7826 #if defined(MARK_ARRAY) && defined(_DEBUG)
7827 size_t cb_end = (size_t)((uint8_t*)card_table_card_bundle_table (ct) + size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address));
7828 #ifdef GROWABLE_SEG_MAPPING_TABLE
7829 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7830 size_t cb_end_aligned = align_for_seg_mapping_table (cb_end);
7831 st += (cb_end_aligned - cb_end);
7832 #else //GROWABLE_SEG_MAPPING_TABLE
7834 #endif //GROWABLE_SEG_MAPPING_TABLE
7835 #endif //MARK_ARRAY && _DEBUG
7836 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7838 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7839 // start of the untranslated table.
7840 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7841 card_table_card_bundle_table (ct));
7843 //set the card table if we are in a heap growth scenario
7844 if (card_bundles_enabled())
7846 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7847 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7849 //check if we need to turn on card_bundles.
7850 #ifdef MULTIPLE_HEAPS
7851 // use INT64 arithmetic here because of possible overflow on 32p
7852 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7854 // use INT64 arithmetic here because of possible overflow on 32p
7855 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7856 #endif //MULTIPLE_HEAPS
7857 if (reserved_memory >= th)
7859 enable_card_bundles();
7862 #endif //CARD_BUNDLE
7864 // for each of the segments and heaps, copy the brick table and
7865 // or the card table
7866 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7869 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7871 //check if it became in range
7872 if ((heap_segment_reserved (seg) > lowest_address) &&
7873 (heap_segment_mem (seg) < highest_address))
7875 set_ro_segment_in_range (seg);
7881 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7882 copy_brick_card_range (la, old_card_table,
7885 align_lower_page (heap_segment_mem (seg)),
7888 seg = heap_segment_next (seg);
7891 seg = generation_start_segment (large_object_generation);
7894 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7896 //check if it became in range
7897 if ((heap_segment_reserved (seg) > lowest_address) &&
7898 (heap_segment_mem (seg) < highest_address))
7900 set_ro_segment_in_range (seg);
7905 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7906 copy_brick_card_range (la, old_card_table,
7909 align_lower_page (heap_segment_mem (seg)),
7912 seg = heap_segment_next (seg);
7915 release_card_table (&old_card_table[card_word (card_of(la))]);
7918 #ifdef FEATURE_BASICFREEZE
7919 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7921 enter_spin_lock (&gc_heap::gc_lock);
7923 if (!gc_heap::seg_table->ensure_space_for_insert ()
7924 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7926 leave_spin_lock(&gc_heap::gc_lock);
7930 //insert at the head of the segment list
7931 generation* gen2 = generation_of (max_generation);
7932 heap_segment* oldhead = generation_start_segment (gen2);
7933 heap_segment_next (seg) = oldhead;
7934 generation_start_segment (gen2) = seg;
7936 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7938 #ifdef SEG_MAPPING_TABLE
7939 seg_mapping_table_add_ro_segment (seg);
7940 #endif //SEG_MAPPING_TABLE
7943 if ((heap_segment_reserved (seg) > lowest_address) &&
7944 (heap_segment_mem (seg) < highest_address))
7946 set_ro_segment_in_range (seg);
7949 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7951 leave_spin_lock (&gc_heap::gc_lock);
7955 // No one is calling this function right now. If this is getting called we need
7956 // to take care of decommitting the mark array for it - we will need to remember
7957 // which portion of the mark array was committed and only decommit that.
7958 void gc_heap::remove_ro_segment (heap_segment* seg)
7960 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7962 if (gc_can_use_concurrent)
7964 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7965 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7966 false); // read_only segments need the mark clear
7970 enter_spin_lock (&gc_heap::gc_lock);
7972 seg_table->remove ((uint8_t*)seg);
7974 #ifdef SEG_MAPPING_TABLE
7975 seg_mapping_table_remove_ro_segment (seg);
7976 #endif //SEG_MAPPING_TABLE
7978 // Locate segment (and previous segment) in the list.
7979 generation* gen2 = generation_of (max_generation);
7980 heap_segment* curr_seg = generation_start_segment (gen2);
7981 heap_segment* prev_seg = NULL;
7983 while (curr_seg && curr_seg != seg)
7985 prev_seg = curr_seg;
7986 curr_seg = heap_segment_next (curr_seg);
7988 assert (curr_seg == seg);
7990 // Patch previous segment (or list head if there is none) to skip the removed segment.
7992 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7994 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7996 leave_spin_lock (&gc_heap::gc_lock);
7998 #endif //FEATURE_BASICFREEZE
8000 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
8003 seg->flags |= heap_segment_flags_inrange;
8004 // init_brick_card_range (seg);
8005 ro_segments_in_range = TRUE;
8006 //right now, segments aren't protected
8007 //unprotect_segment (seg);
8013 uint8_t** make_mark_list (size_t size)
8015 uint8_t** mark_list = new (nothrow) uint8_t* [size];
8019 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
8021 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
8025 for (i = low+1; i <= high; i++)
8034 #ifndef USE_INTROSORT
8035 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
8037 if (((low + 16) >= high) || (depth > 100))
8041 for (i = low+1; i <= high; i++)
8044 for (j=i;j >low && val<*(j-1);j--)
8053 uint8_t *pivot, **left, **right;
8055 //sort low middle and high
8056 if (*(low+((high-low)/2)) < *low)
8057 swap (*(low+((high-low)/2)), *low);
8060 if (*high < *(low+((high-low)/2)))
8061 swap (*(low+((high-low)/2)), *high);
8063 swap (*(low+((high-low)/2)), *(high-1));
8065 left = low; right = high-1;
8067 while (*(--right) > pivot);
8068 while (*(++left) < pivot);
8071 swap(*left, *right);
8076 swap (*left, *(high-1));
8077 qsort1(low, left-1, depth+1);
8078 qsort1(left+1, high, depth+1);
8081 #endif //USE_INTROSORT
8082 void rqsort1( uint8_t* *low, uint8_t* *high)
8084 if ((low + 16) >= high)
8088 for (i = low+1; i <= high; i++)
8091 for (j=i;j >low && val>*(j-1);j--)
8100 uint8_t *pivot, **left, **right;
8102 //sort low middle and high
8103 if (*(low+((high-low)/2)) > *low)
8104 swap (*(low+((high-low)/2)), *low);
8107 if (*high > *(low+((high-low)/2)))
8108 swap (*(low+((high-low)/2)), *high);
8110 swap (*(low+((high-low)/2)), *(high-1));
8112 left = low; right = high-1;
8114 while (*(--right) < pivot);
8115 while (*(++left) > pivot);
8118 swap(*left, *right);
8123 swap (*left, *(high-1));
8124 rqsort1(low, left-1);
8125 rqsort1(left+1, high);
8129 #ifdef USE_INTROSORT
8134 static const int size_threshold = 64;
8135 static const int max_depth = 100;
8138 inline static void swap_elements(uint8_t** i,uint8_t** j)
8146 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8149 introsort_loop (begin, end, max_depth);
8150 insertionsort (begin, end);
8155 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8157 while (hi-lo >= size_threshold)
8159 if (depth_limit == 0)
8164 uint8_t** p=median_partition (lo, hi);
8165 depth_limit=depth_limit-1;
8166 introsort_loop (p, hi, depth_limit);
8171 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8173 uint8_t *pivot, **left, **right;
8175 //sort low middle and high
8176 if (*(low+((high-low)/2)) < *low)
8177 swap_elements ((low+((high-low)/2)), low);
8179 swap_elements (low, high);
8180 if (*high < *(low+((high-low)/2)))
8181 swap_elements ((low+((high-low)/2)), high);
8183 swap_elements ((low+((high-low)/2)), (high-1));
8185 left = low; right = high-1;
8187 while (*(--right) > pivot);
8188 while (*(++left) < pivot);
8191 swap_elements(left, right);
8196 swap_elements (left, (high-1));
8201 static void insertionsort (uint8_t** lo, uint8_t** hi)
8203 for (uint8_t** i=lo+1; i <= hi; i++)
8207 while((j > lo) && (t <*(j-1)))
8216 static void heapsort (uint8_t** lo, uint8_t** hi)
8218 size_t n = hi - lo + 1;
8219 for (size_t i=n / 2; i >= 1; i--)
8223 for (size_t i = n; i > 1; i--)
8225 swap_elements (lo, lo + i - 1);
8226 downheap(1, i - 1, lo);
8230 static void downheap (size_t i, size_t n, uint8_t** lo)
8232 uint8_t* d = *(lo + i - 1);
8237 if (child < n && *(lo + child - 1)<(*(lo + child)))
8241 if (!(d<*(lo + child - 1)))
8245 *(lo + i - 1) = *(lo + child - 1);
8253 #endif //USE_INTROSORT
8255 #ifdef MULTIPLE_HEAPS
8256 #ifdef PARALLEL_MARK_LIST_SORT
8257 void gc_heap::sort_mark_list()
8259 // if this heap had a mark list overflow, we don't do anything
8260 if (mark_list_index > mark_list_end)
8262 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8266 // if any other heap had a mark list overflow, we fake one too,
8267 // so we don't use an incomplete mark list by mistake
8268 for (int i = 0; i < n_heaps; i++)
8270 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8272 mark_list_index = mark_list_end + 1;
8273 // printf("sort_mark_list: overflow on heap %d\n", i);
8278 // unsigned long start = GetCycleCount32();
8280 dprintf (3, ("Sorting mark lists"));
8281 if (mark_list_index > mark_list)
8282 _sort (mark_list, mark_list_index - 1, 0);
8284 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8285 // start = GetCycleCount32();
8287 // first set the pieces for all heaps to empty
8289 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8291 mark_list_piece_start[heap_num] = NULL;
8292 mark_list_piece_end[heap_num] = NULL;
8295 uint8_t** x = mark_list;
8297 // predicate means: x is still within the mark list, and within the bounds of this heap
8298 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8301 while (x < mark_list_index)
8304 // find the heap x points into - searching cyclically from the last heap,
8305 // because in many cases the right heap is the next one or comes soon after
8306 int last_heap_num = heap_num;
8307 MAYBE_UNUSED_VAR(last_heap_num);
8311 if (heap_num >= n_heaps)
8313 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8314 heap = g_heaps[heap_num];
8316 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8318 // x is the start of the mark list piece for this heap
8319 mark_list_piece_start[heap_num] = x;
8321 // to find the end of the mark list piece for this heap, find the first x
8322 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8325 // let's see if we get lucky and the whole rest belongs to this piece
8326 if (predicate(mark_list_index-1))
8328 x = mark_list_index;
8329 mark_list_piece_end[heap_num] = x;
8333 // we play a variant of binary search to find the point sooner.
8334 // the first loop advances by increasing steps until the predicate turns false.
8335 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8340 uint8_t** temp_x = x;
8347 while (predicate(x));
8348 // we know that only the last step was wrong, so we undo it
8352 // loop invariant - predicate holds at x, but not x + inc
8353 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8355 if (((x + inc) > x) && predicate(x + inc))
8361 // the termination condition and the loop invariant together imply this:
8362 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8363 // so the spot we're looking for is one further
8366 mark_list_piece_end[heap_num] = x;
8371 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8374 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8376 size_t slots_needed = end - start;
8377 size_t slots_available = mark_list_end + 1 - mark_list_index;
8378 size_t slots_to_copy = min(slots_needed, slots_available);
8379 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8380 mark_list_index += slots_to_copy;
8381 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8384 void gc_heap::merge_mark_lists()
8386 uint8_t** source[MAX_SUPPORTED_CPUS];
8387 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8388 int source_heap[MAX_SUPPORTED_CPUS];
8389 int source_count = 0;
8391 // in case of mark list overflow, don't bother
8392 if (mark_list_index > mark_list_end)
8394 // printf("merge_mark_lists: overflow\n");
8398 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8399 // unsigned long start = GetCycleCount32();
8400 for (int i = 0; i < n_heaps; i++)
8402 gc_heap* heap = g_heaps[i];
8403 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8405 source[source_count] = heap->mark_list_piece_start[heap_number];
8406 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8407 source_heap[source_count] = i;
8408 if (source_count < MAX_SUPPORTED_CPUS)
8412 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8414 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8415 #if defined(_DEBUG) || defined(TRACE_GC)
8416 for (int j = 0; j < source_count; j++)
8418 dprintf(3, ("heap_number = %d ", heap_number));
8419 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8420 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8421 // the sources should all be sorted
8422 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8426 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8431 #endif //_DEBUG || TRACE_GC
8433 // start = GetCycleCount32();
8435 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8436 mark_list_index = mark_list;
8437 mark_list_end = &mark_list [mark_list_size-1];
8438 int piece_count = 0;
8439 if (source_count == 0)
8443 else if (source_count == 1)
8445 mark_list = source[0];
8446 mark_list_index = source_end[0];
8447 mark_list_end = mark_list_index;
8452 while (source_count > 1)
8454 // find the lowest and second lowest value in the sources we're merging from
8455 int lowest_source = 0;
8456 uint8_t *lowest = *source[0];
8457 uint8_t *second_lowest = *source[1];
8458 for (int i = 1; i < source_count; i++)
8460 if (lowest > *source[i])
8462 second_lowest = lowest;
8463 lowest = *source[i];
8466 else if (second_lowest > *source[i])
8468 second_lowest = *source[i];
8472 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8474 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8476 if (source_end[lowest_source][-1] <= second_lowest)
8477 x = source_end[lowest_source];
8480 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8481 // but saw no improvement doing that
8482 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8486 // blast this piece to the mark list
8487 append_to_mark_list(source[lowest_source], x);
8490 source[lowest_source] = x;
8492 // check whether this source is now exhausted
8493 if (x >= source_end[lowest_source])
8495 // if it's not the source with the highest index, copy the source with the highest index
8496 // over it so the non-empty sources are always at the beginning
8497 if (lowest_source < source_count-1)
8499 source[lowest_source] = source[source_count-1];
8500 source_end[lowest_source] = source_end[source_count-1];
8505 // we're left with just one source that we copy
8506 append_to_mark_list(source[0], source_end[0]);
8510 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8512 #if defined(_DEBUG) || defined(TRACE_GC)
8513 // the final mark list must be sorted
8514 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8518 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8522 #endif //defined(_DEBUG) || defined(TRACE_GC)
8524 #else //PARALLEL_MARK_LIST_SORT
8525 void gc_heap::combine_mark_lists()
8527 dprintf (3, ("Combining mark lists"));
8528 //verify if a heap has overflowed its mark list
8529 BOOL use_mark_list = TRUE;
8530 for (int i = 0; i < n_heaps; i++)
8532 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8534 use_mark_list = FALSE;
8541 dprintf (3, ("Using mark list"));
8542 //compact the gaps out of the mark list
8544 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8545 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8546 uint8_t** dst_last = current_gap-1;
8548 int srcn = n_heaps-1;
8549 gc_heap* srch = g_heaps [srcn];
8550 uint8_t** src = srch->mark_list_index - 1;
8551 uint8_t** src_beg = srch->mark_list;
8553 while (current_gap <= src)
8555 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8557 //go to the next gap
8559 dprintf (3, ("Going to the next gap %d", gn));
8560 assert (gn < n_heaps);
8561 current_gap = g_heaps [gn]->mark_list_index;
8562 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8563 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8565 while ((srcn > 0) && (src < src_beg))
8567 //go to the previous source
8569 dprintf (3, ("going to the previous source %d", srcn));
8571 gc_heap* srch = g_heaps [srcn];
8572 src = srch->mark_list_index - 1;
8573 src_beg = srch->mark_list;
8575 if (current_gap < src)
8577 dst_last = current_gap;
8578 *current_gap++ = *src--;
8581 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8583 uint8_t** end_of_list = max (src, dst_last);
8585 //sort the resulting compacted list
8586 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8587 if (end_of_list > &g_mark_list[0])
8588 _sort (&g_mark_list[0], end_of_list, 0);
8589 //adjust the mark_list to the begining of the resulting mark list.
8590 for (int i = 0; i < n_heaps; i++)
8592 g_heaps [i]->mark_list = g_mark_list;
8593 g_heaps [i]->mark_list_index = end_of_list + 1;
8594 g_heaps [i]->mark_list_end = end_of_list + 1;
8599 uint8_t** end_of_list = g_mark_list;
8600 //adjust the mark_list to the begining of the resulting mark list.
8601 //put the index beyond the end to turn off mark list processing
8602 for (int i = 0; i < n_heaps; i++)
8604 g_heaps [i]->mark_list = g_mark_list;
8605 g_heaps [i]->mark_list_index = end_of_list + 1;
8606 g_heaps [i]->mark_list_end = end_of_list;
8610 #endif // PARALLEL_MARK_LIST_SORT
8611 #endif //MULTIPLE_HEAPS
8614 class seg_free_spaces
8616 struct seg_free_space
8622 struct free_space_bucket
8624 seg_free_space* free_space;
8625 ptrdiff_t count_add; // Assigned when we first contruct the array.
8626 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8629 void move_bucket (int old_power2, int new_power2)
8631 // PREFAST warning 22015: old_power2 could be negative
8632 assert (old_power2 >= 0);
8633 assert (old_power2 >= new_power2);
8635 if (old_power2 == new_power2)
8640 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8641 for (int i = old_power2; i > new_power2; i--)
8643 seg_free_space** dest = &(free_space_buckets[i].free_space);
8646 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8647 if (i > (new_power2 + 1))
8649 seg_free_space temp = *src_index;
8650 *src_index = *dest_index;
8653 src_index = dest_index;
8656 free_space_buckets[old_power2].count_fit--;
8657 free_space_buckets[new_power2].count_fit++;
8662 void dump_free_space (seg_free_space* item)
8669 mark* m = (mark*)(item->start);
8670 len = pinned_len (m);
8671 addr = pinned_plug (m) - len;
8675 heap_segment* seg = (heap_segment*)(item->start);
8676 addr = heap_segment_plan_allocated (seg);
8677 len = heap_segment_committed (seg) - addr;
8680 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8685 seg_free_space* item = NULL;
8688 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8689 for (i = 0; i < (free_space_bucket_count - 1); i++)
8691 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8692 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8693 item = free_space_buckets[i].free_space;
8694 while (item < free_space_buckets[i + 1].free_space)
8696 dump_free_space (item);
8699 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8702 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8703 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8704 item = free_space_buckets[i].free_space;
8706 while (item <= &seg_free_space_array[free_space_item_count - 1])
8708 dump_free_space (item);
8711 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8716 free_space_bucket* free_space_buckets;
8717 seg_free_space* seg_free_space_array;
8718 ptrdiff_t free_space_bucket_count;
8719 ptrdiff_t free_space_item_count;
8723 BOOL has_end_of_seg;
8728 seg_free_spaces (int h_number)
8730 heap_num = h_number;
8735 size_t total_prealloc_size =
8736 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8737 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8739 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8741 return (!!free_space_buckets);
8744 // We take the ordered free space array we got from the 1st pass,
8745 // and feed the portion that we decided to use to this method, ie,
8746 // the largest item_count free spaces.
8747 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8749 assert (free_space_buckets);
8750 assert (item_count <= (size_t)MAX_PTR);
8752 free_space_bucket_count = bucket_count;
8753 free_space_item_count = item_count;
8756 has_end_of_seg = FALSE;
8759 ptrdiff_t total_item_count = 0;
8762 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8764 for (i = 0; i < (ptrdiff_t)item_count; i++)
8766 seg_free_space_array[i].start = 0;
8767 seg_free_space_array[i].is_plug = FALSE;
8770 for (i = 0; i < bucket_count; i++)
8772 free_space_buckets[i].count_add = ordered_free_spaces[i];
8773 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8774 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8775 total_item_count += free_space_buckets[i].count_add;
8778 assert (total_item_count == (ptrdiff_t)item_count);
8781 // If we are adding a free space before a plug we pass the
8782 // mark stack position so we can update the length; we could
8783 // also be adding the free space after the last plug in which
8784 // case start is the segment which we'll need to update the
8785 // heap_segment_plan_allocated.
8786 void add (void* start, BOOL plug_p, BOOL first_p)
8788 size_t size = (plug_p ?
8789 pinned_len ((mark*)start) :
8790 (heap_segment_committed ((heap_segment*)start) -
8791 heap_segment_plan_allocated ((heap_segment*)start)));
8795 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8799 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8801 has_end_of_seg = TRUE;
8807 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8808 size -= eph_gen_starts;
8811 mark* m = (mark*)(start);
8812 pinned_len (m) -= eph_gen_starts;
8816 heap_segment* seg = (heap_segment*)start;
8817 heap_segment_plan_allocated (seg) += eph_gen_starts;
8821 int bucket_power2 = index_of_highest_set_bit (size);
8822 if (bucket_power2 < base_power2)
8827 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8829 seg_free_space* bucket_free_space = bucket->free_space;
8830 assert (plug_p || (!plug_p && bucket->count_add));
8832 if (bucket->count_add == 0)
8834 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8838 ptrdiff_t index = bucket->count_add - 1;
8840 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8843 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8844 heap_segment_plan_allocated ((heap_segment*)start)),
8850 bucket_free_space[index].is_plug = TRUE;
8853 bucket_free_space[index].start = start;
8854 bucket->count_add--;
8859 // Do a consistency check after all free spaces are added.
8863 int end_of_seg_count = 0;
8865 for (i = 0; i < free_space_item_count; i++)
8867 assert (seg_free_space_array[i].start);
8868 if (!(seg_free_space_array[i].is_plug))
8876 assert (end_of_seg_count == 1);
8880 assert (end_of_seg_count == 0);
8883 for (i = 0; i < free_space_bucket_count; i++)
8885 assert (free_space_buckets[i].count_add == 0);
8891 uint8_t* fit (uint8_t* old_loc,
8893 BOOL set_padding_on_saved_p,
8894 mark* pinned_plug_entry,
8895 #endif //SHORT_PLUGS
8897 REQD_ALIGN_AND_OFFSET_DCL)
8902 assert (!is_plug_padded (old_loc));
8903 #endif //SHORT_PLUGS
8904 assert (!node_realigned (old_loc));
8907 size_t saved_plug_size = plug_size;
8909 #ifdef FEATURE_STRUCTALIGN
8910 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8911 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8912 #endif // FEATURE_STRUCTALIGN
8913 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8916 size_t plug_size_to_fit = plug_size;
8918 // best fit is only done for gen1 to gen2 and we do not pad in gen2.
8919 int pad_in_front = 0;
8922 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8923 #endif //SHORT_PLUGS
8925 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8927 uint8_t* new_address = 0;
8929 if (plug_power2 < base_power2)
8931 plug_power2 = base_power2;
8934 int chosen_power2 = plug_power2 - base_power2;
8936 for (i = chosen_power2; i < free_space_bucket_count; i++)
8938 if (free_space_buckets[i].count_fit != 0)
8945 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8949 (chosen_power2 + base_power2)));
8951 assert (i < free_space_bucket_count);
8953 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8954 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8955 size_t new_free_space_size = 0;
8956 BOOL can_fit = FALSE;
8959 for (i = 0; i < free_space_count; i++)
8961 size_t free_space_size = 0;
8964 BOOL short_plugs_padding_p = FALSE;
8965 #endif //SHORT_PLUGS
8966 BOOL realign_padding_p = FALSE;
8968 if (bucket_free_space[i].is_plug)
8970 mark* m = (mark*)(bucket_free_space[i].start);
8971 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8974 if ((pad_in_front & USE_PADDING_FRONT) &&
8975 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8976 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8978 pad = Align (min_obj_size);
8979 short_plugs_padding_p = TRUE;
8981 #endif //SHORT_PLUGS
8983 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8985 pad += switch_alignment_size (pad != 0);
8986 realign_padding_p = TRUE;
8989 plug_size = saved_plug_size + pad;
8991 free_space_size = pinned_len (m);
8992 new_address = pinned_plug (m) - pinned_len (m);
8994 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8995 free_space_size == plug_size)
8997 new_free_space_size = free_space_size - plug_size;
8998 pinned_len (m) = new_free_space_size;
8999 #ifdef SIMPLE_DPRINTF
9000 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
9007 index_of_highest_set_bit (free_space_size),
9008 (pinned_plug (m) - pinned_len (m)),
9009 index_of_highest_set_bit (new_free_space_size)));
9010 #endif //SIMPLE_DPRINTF
9013 if (short_plugs_padding_p)
9015 pin_allocation_context_start_region (m) = plug_free_space_start;
9016 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
9018 #endif //SHORT_PLUGS
9020 if (realign_padding_p)
9022 set_node_realigned (old_loc);
9030 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
9031 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
9033 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
9035 pad = switch_alignment_size (FALSE);
9036 realign_padding_p = TRUE;
9039 plug_size = saved_plug_size + pad;
9041 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
9042 free_space_size == plug_size)
9044 new_address = heap_segment_plan_allocated (seg);
9045 new_free_space_size = free_space_size - plug_size;
9046 heap_segment_plan_allocated (seg) = new_address + plug_size;
9047 #ifdef SIMPLE_DPRINTF
9048 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
9053 index_of_highest_set_bit (free_space_size),
9054 heap_segment_plan_allocated (seg),
9055 index_of_highest_set_bit (new_free_space_size)));
9056 #endif //SIMPLE_DPRINTF
9058 if (realign_padding_p)
9059 set_node_realigned (old_loc);
9073 assert (chosen_power2 == 0);
9083 assert ((chosen_power2 && (i == 0)) ||
9084 ((!chosen_power2) && (i < free_space_count)));
9087 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
9089 if (new_bucket_power2 < base_power2)
9091 new_bucket_power2 = base_power2;
9094 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
9103 if (free_space_buckets)
9105 delete [] free_space_buckets;
9107 if (seg_free_space_array)
9109 delete [] seg_free_space_array;
9115 #define marked(i) header(i)->IsMarked()
9116 #define set_marked(i) header(i)->SetMarked()
9117 #define clear_marked(i) header(i)->ClearMarked()
9118 #define pinned(i) header(i)->IsPinned()
9119 #define set_pinned(i) header(i)->SetPinned()
9120 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
9122 inline size_t my_get_size (Object* ob)
9124 MethodTable* mT = header(ob)->GetMethodTable();
9125 return (mT->GetBaseSize() +
9126 (mT->HasComponentSize() ?
9127 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
9130 //#define size(i) header(i)->GetSize()
9131 #define size(i) my_get_size (header(i))
9133 #define contain_pointers(i) header(i)->ContainsPointers()
9134 #ifdef COLLECTIBLE_CLASS
9135 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9137 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9138 #define is_collectible(i) method_table(i)->Collectible()
9139 #else //COLLECTIBLE_CLASS
9140 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9141 #endif //COLLECTIBLE_CLASS
9143 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9145 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9147 uint8_t* range_beg = 0;
9148 uint8_t* range_end = 0;
9149 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9151 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9152 #ifdef FEATURE_BASICFREEZE
9154 #endif // FEATURE_BASICFREEZE
9159 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9161 if ((start < background_saved_highest_address) &&
9162 (end > background_saved_lowest_address))
9164 start = max (start, background_saved_lowest_address);
9165 end = min (end, background_saved_highest_address);
9167 size_t start_mark_bit = mark_bit_of (start);
9168 size_t end_mark_bit = mark_bit_of (end);
9169 unsigned int startbit = mark_bit_bit (start_mark_bit);
9170 unsigned int endbit = mark_bit_bit (end_mark_bit);
9171 size_t startwrd = mark_bit_word (start_mark_bit);
9172 size_t endwrd = mark_bit_word (end_mark_bit);
9174 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9175 (size_t)start, (size_t)start_mark_bit,
9176 (size_t)end, (size_t)end_mark_bit));
9178 unsigned int firstwrd = lowbits (~0, startbit);
9179 unsigned int lastwrd = highbits (~0, endbit);
9181 if (startwrd == endwrd)
9183 unsigned int wrd = firstwrd | lastwrd;
9184 mark_array[startwrd] &= wrd;
9188 // clear the first mark word.
9191 mark_array[startwrd] &= firstwrd;
9195 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9197 mark_array[wrdtmp] = 0;
9200 // clear the last mark word.
9203 mark_array[endwrd] &= lastwrd;
9208 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9210 if ((start < background_saved_highest_address) &&
9211 (end > background_saved_lowest_address))
9213 start = max (start, background_saved_lowest_address);
9214 end = min (end, background_saved_highest_address);
9216 clear_batch_mark_array_bits (start, end);
9220 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9222 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9224 int align_const = get_alignment_constant (!loh_p);
9230 uint8_t* next_o = o + Align (size (o), align_const);
9232 if (background_object_marked (o, TRUE))
9234 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9240 #endif //MARK_ARRAY && BACKGROUND_GC
9243 BOOL gc_heap::is_mark_set (uint8_t* o)
9248 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9249 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9250 #endif //_MSC_VER && _TARGET_X86_
9252 // return the generation number of an object.
9253 // It is assumed that the object is valid.
9254 //Note that this will return max_generation for a LOH object
9255 int gc_heap::object_gennum (uint8_t* o)
9257 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9258 (o >= generation_allocation_start (generation_of (max_generation-1))))
9260 // in an ephemeral generation.
9261 for ( int i = 0; i < max_generation-1; i++)
9263 if ((o >= generation_allocation_start (generation_of (i))))
9266 return max_generation-1;
9270 return max_generation;
9274 int gc_heap::object_gennum_plan (uint8_t* o)
9276 if (in_range_for_segment (o, ephemeral_heap_segment))
9278 for (int i = 0; i <= max_generation-1; i++)
9280 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9281 if (plan_start && (o >= plan_start))
9287 return max_generation;
9290 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9291 #pragma optimize("", on) // Go back to command line default optimizations
9292 #endif //_MSC_VER && _TARGET_X86_
9294 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9296 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9298 //Commit the first page
9299 if (!virtual_commit (new_pages, initial_commit, h_number))
9304 //overlay the heap_segment
9305 heap_segment* new_segment = (heap_segment*)new_pages;
9307 uint8_t* start = new_pages + segment_info_size;
9308 heap_segment_mem (new_segment) = start;
9309 heap_segment_used (new_segment) = start;
9310 heap_segment_reserved (new_segment) = new_pages + size;
9311 heap_segment_committed (new_segment) = new_pages + initial_commit;
9312 init_heap_segment (new_segment);
9313 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9317 void gc_heap::init_heap_segment (heap_segment* seg)
9320 heap_segment_next (seg) = 0;
9321 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9322 heap_segment_allocated (seg) = heap_segment_mem (seg);
9323 #ifdef BACKGROUND_GC
9324 heap_segment_background_allocated (seg) = 0;
9325 heap_segment_saved_bg_allocated (seg) = 0;
9326 #endif //BACKGROUND_GC
9329 //Releases the segment to the OS.
9330 // this is always called on one thread only so calling seg_table->remove is fine.
9331 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9333 if (!heap_segment_loh_p (seg))
9335 //cleanup the brick table back to the empty value
9336 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9339 if (consider_hoarding)
9341 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9342 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9343 //Don't keep the big ones.
9344 if (ss <= INITIAL_ALLOC)
9346 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9347 #ifdef BACKGROUND_GC
9348 // We don't need to clear the decommitted flag because when this segment is used
9349 // for a new segment the flags will be cleared.
9350 if (!heap_segment_decommitted_p (seg))
9351 #endif //BACKGROUND_GC
9353 decommit_heap_segment (seg);
9356 #ifdef SEG_MAPPING_TABLE
9357 seg_mapping_table_remove_segment (seg);
9358 #endif //SEG_MAPPING_TABLE
9360 heap_segment_next (seg) = segment_standby_list;
9361 segment_standby_list = seg;
9368 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9369 heap_number, (size_t)seg,
9370 (size_t)(heap_segment_reserved (seg))));
9372 #ifdef BACKGROUND_GC
9373 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9374 settings.gc_index, current_bgc_state,
9376 decommit_mark_array_by_seg (seg);
9377 #endif //BACKGROUND_GC
9379 #ifdef SEG_MAPPING_TABLE
9380 seg_mapping_table_remove_segment (seg);
9381 #else //SEG_MAPPING_TABLE
9382 seg_table->remove ((uint8_t*)seg);
9383 #endif //SEG_MAPPING_TABLE
9385 release_segment (seg);
9389 //resets the pages beyond allocates size so they won't be swapped out and back in
9391 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9393 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9394 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9396 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9399 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9402 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9403 size_t size = heap_segment_committed (seg) - page_start;
9404 extra_space = align_on_page (extra_space);
9405 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9407 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9408 size -= max (extra_space, 32*OS_PAGE_SIZE);
9410 virtual_decommit (page_start, size, heap_number);
9411 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9413 (size_t)(page_start + size),
9415 heap_segment_committed (seg) = page_start;
9416 if (heap_segment_used (seg) > heap_segment_committed (seg))
9418 heap_segment_used (seg) = heap_segment_committed (seg);
9423 //decommit all pages except one or 2
9424 void gc_heap::decommit_heap_segment (heap_segment* seg)
9426 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9428 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9430 #ifdef BACKGROUND_GC
9431 page_start += OS_PAGE_SIZE;
9432 #endif //BACKGROUND_GC
9434 size_t size = heap_segment_committed (seg) - page_start;
9435 virtual_decommit (page_start, size, heap_number);
9437 //re-init the segment object
9438 heap_segment_committed (seg) = page_start;
9439 if (heap_segment_used (seg) > heap_segment_committed (seg))
9441 heap_segment_used (seg) = heap_segment_committed (seg);
9445 void gc_heap::clear_gen0_bricks()
9447 if (!gen0_bricks_cleared)
9449 gen0_bricks_cleared = TRUE;
9450 //initialize brick table for gen 0
9451 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9452 b < brick_of (align_on_brick
9453 (heap_segment_allocated (ephemeral_heap_segment)));
9461 #ifdef BACKGROUND_GC
9462 void gc_heap::rearrange_small_heap_segments()
9464 heap_segment* seg = freeable_small_heap_segment;
9467 heap_segment* next_seg = heap_segment_next (seg);
9468 // TODO: we need to consider hoarding here.
9469 delete_heap_segment (seg, FALSE);
9472 freeable_small_heap_segment = 0;
9474 #endif //BACKGROUND_GC
9476 void gc_heap::rearrange_large_heap_segments()
9478 dprintf (2, ("deleting empty large segments"));
9479 heap_segment* seg = freeable_large_heap_segment;
9482 heap_segment* next_seg = heap_segment_next (seg);
9483 delete_heap_segment (seg, GCConfig::GetRetainVM());
9486 freeable_large_heap_segment = 0;
9489 void gc_heap::rearrange_heap_segments(BOOL compacting)
9492 generation_start_segment (generation_of (max_generation));
9494 heap_segment* prev_seg = 0;
9495 heap_segment* next_seg = 0;
9498 next_seg = heap_segment_next (seg);
9500 //link ephemeral segment when expanding
9501 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9503 seg->next = ephemeral_heap_segment;
9504 next_seg = heap_segment_next (seg);
9507 //re-used expanded heap segment
9508 if ((seg == ephemeral_heap_segment) && next_seg)
9510 heap_segment_next (prev_seg) = next_seg;
9511 heap_segment_next (seg) = 0;
9515 uint8_t* end_segment = (compacting ?
9516 heap_segment_plan_allocated (seg) :
9517 heap_segment_allocated (seg));
9518 // check if the segment was reached by allocation
9519 if ((end_segment == heap_segment_mem (seg))&&
9520 !heap_segment_read_only_p (seg))
9522 //if not, unthread and delete
9524 assert (seg != ephemeral_heap_segment);
9525 heap_segment_next (prev_seg) = next_seg;
9526 delete_heap_segment (seg, GCConfig::GetRetainVM());
9528 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9532 if (!heap_segment_read_only_p (seg))
9536 heap_segment_allocated (seg) =
9537 heap_segment_plan_allocated (seg);
9540 // reset the pages between allocated and committed.
9541 if (seg != ephemeral_heap_segment)
9543 decommit_heap_segment_pages (seg, 0);
9557 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9559 #ifdef TIME_WRITE_WATCH
9560 static unsigned int tot_cycles = 0;
9561 #endif //TIME_WRITE_WATCH
9565 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9568 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9570 if (!card_bundle_set_p (x))
9572 assert (!"Card bundle not set");
9573 dprintf (3, ("Card bundle %Ix not set", x));
9579 // Verifies that any bundles that are not set represent only cards that are not set.
9580 inline void gc_heap::verify_card_bundles()
9583 size_t lowest_card = card_word (card_of (lowest_address));
9584 size_t highest_card = card_word (card_of (highest_address));
9585 size_t cardb = cardw_card_bundle (lowest_card);
9586 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9588 while (cardb < end_cardb)
9590 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9591 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9593 if (card_bundle_set_p (cardb) == 0)
9595 // Verify that no card is set
9596 while (card_word < card_word_end)
9598 if (*card_word != 0)
9600 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9601 dd_collection_count (dynamic_data_of (0)),
9602 (size_t)(card_word-&card_table[0]),
9603 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9606 assert((*card_word)==0);
9616 // If card bundles are enabled, use write watch to find pages in the card table that have
9617 // been dirtied, and set the corresponding card bundle bits.
9618 void gc_heap::update_card_table_bundle()
9620 if (card_bundles_enabled())
9622 // The address of the card word containing the card representing the lowest heap address
9623 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9625 // The address of the card word containing the card representing the highest heap address
9626 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9628 uint8_t* saved_base_address = base_address;
9629 uintptr_t bcount = array_size;
9630 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9634 size_t region_size = align_on_page (high_address) - base_address;
9636 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9637 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9640 (void**)g_addresses,
9642 assert (success && "GetWriteWatch failed!");
9644 dprintf (3,("Found %d pages written", bcount));
9645 for (unsigned i = 0; i < bcount; i++)
9647 // Offset of the dirty page from the start of the card table (clamped to base_address)
9648 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9650 // Offset of the end of the page from the start of the card table (clamped to high addr)
9651 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9652 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9654 // Set the card bundle bits representing the dirty card table page
9655 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9656 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9658 verify_card_bundle_bits_set(bcardw, ecardw);
9661 if (bcount >= array_size)
9663 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9664 bcount = array_size;
9667 } while ((bcount >= array_size) && (base_address < high_address));
9669 // Now that we've updated the card bundle bits, reset the write-tracking state.
9670 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9673 #endif //CARD_BUNDLE
9676 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9678 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9679 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9680 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9681 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9682 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9686 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9688 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9689 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9690 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9691 UNREFERENCED_PARAMETER(is_runtime_suspended);
9692 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9694 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9697 const size_t ww_reset_quantum = 128*1024*1024;
9700 void gc_heap::switch_one_quantum()
9702 enable_preemptive ();
9703 GCToOSInterface::Sleep (1);
9704 disable_preemptive (true);
9707 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9709 size_t reset_size = 0;
9710 size_t remaining_reset_size = 0;
9711 size_t next_reset_size = 0;
9713 while (reset_size != total_reset_size)
9715 remaining_reset_size = total_reset_size - reset_size;
9716 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9717 if (next_reset_size)
9719 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9720 reset_size += next_reset_size;
9722 switch_one_quantum();
9726 assert (reset_size == total_reset_size);
9729 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9730 // we do concurrently.
9731 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9735 *current_total_reset_size += last_reset_size;
9737 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9739 if (*current_total_reset_size > ww_reset_quantum)
9741 switch_one_quantum();
9743 *current_total_reset_size = 0;
9748 void gc_heap::reset_write_watch (BOOL concurrent_p)
9750 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9751 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9752 assert(!concurrent_p);
9753 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9755 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9757 PREFIX_ASSUME(seg != NULL);
9759 size_t reset_size = 0;
9760 size_t region_size = 0;
9762 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9766 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9767 base_address = max (base_address, background_saved_lowest_address);
9769 uint8_t* high_address = 0;
9770 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9771 high_address = min (high_address, background_saved_highest_address);
9773 if (base_address < high_address)
9775 region_size = high_address - base_address;
9777 #ifdef TIME_WRITE_WATCH
9778 unsigned int time_start = GetCycleCount32();
9779 #endif //TIME_WRITE_WATCH
9780 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9781 //reset_ww_by_chunk (base_address, region_size);
9782 reset_write_watch_for_gc_heap(base_address, region_size);
9784 #ifdef TIME_WRITE_WATCH
9785 unsigned int time_stop = GetCycleCount32();
9786 tot_cycles += time_stop - time_start;
9787 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9788 time_stop - time_start, tot_cycles);
9789 #endif //TIME_WRITE_WATCH
9791 switch_on_reset (concurrent_p, &reset_size, region_size);
9794 seg = heap_segment_next_rw (seg);
9796 concurrent_print_time_delta ("CRWW soh");
9799 //concurrent_print_time_delta ("CRW soh");
9801 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9803 PREFIX_ASSUME(seg != NULL);
9807 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9808 uint8_t* high_address = heap_segment_allocated (seg);
9810 base_address = max (base_address, background_saved_lowest_address);
9811 high_address = min (high_address, background_saved_highest_address);
9813 if (base_address < high_address)
9815 region_size = high_address - base_address;
9817 #ifdef TIME_WRITE_WATCH
9818 unsigned int time_start = GetCycleCount32();
9819 #endif //TIME_WRITE_WATCH
9820 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9821 //reset_ww_by_chunk (base_address, region_size);
9822 reset_write_watch_for_gc_heap(base_address, region_size);
9824 #ifdef TIME_WRITE_WATCH
9825 unsigned int time_stop = GetCycleCount32();
9826 tot_cycles += time_stop - time_start;
9827 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9828 time_stop - time_start, tot_cycles);
9829 #endif //TIME_WRITE_WATCH
9831 switch_on_reset (concurrent_p, &reset_size, region_size);
9834 seg = heap_segment_next_rw (seg);
9836 concurrent_print_time_delta ("CRWW loh");
9839 #ifdef DEBUG_WRITE_WATCH
9840 debug_write_watch = (uint8_t**)~0;
9841 #endif //DEBUG_WRITE_WATCH
9844 #endif //WRITE_WATCH
9846 #ifdef BACKGROUND_GC
9847 void gc_heap::restart_vm()
9849 //assert (generation_allocation_pointer (youngest_generation) == 0);
9850 dprintf (3, ("Restarting EE"));
9851 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9852 ee_proceed_event.Set();
9856 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9858 if (awr != awr_ignored)
9862 FIRE_EVENT(BGCAllocWaitBegin, awr);
9866 FIRE_EVENT(BGCAllocWaitEnd, awr);
9872 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9874 fire_alloc_wait_event (awr, TRUE);
9878 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9880 fire_alloc_wait_event (awr, FALSE);
9882 #endif //BACKGROUND_GC
9883 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9885 gen.allocation_start = start;
9886 gen.allocation_context.alloc_ptr = pointer;
9887 gen.allocation_context.alloc_limit = pointer;
9888 gen.allocation_context.alloc_bytes = 0;
9889 gen.allocation_context.alloc_bytes_loh = 0;
9890 gen.allocation_context_start_region = pointer;
9891 gen.start_segment = seg;
9892 gen.allocation_segment = seg;
9893 gen.plan_allocation_start = 0;
9894 gen.free_list_space = 0;
9895 gen.pinned_allocated = 0;
9896 gen.free_list_allocated = 0;
9897 gen.end_seg_allocated = 0;
9898 gen.condemned_allocated = 0;
9899 gen.free_obj_space = 0;
9900 gen.allocation_size = 0;
9901 gen.pinned_allocation_sweep_size = 0;
9902 gen.pinned_allocation_compact_size = 0;
9903 gen.allocate_end_seg_p = FALSE;
9904 gen.free_list_allocator.clear();
9906 #ifdef FREE_USAGE_STATS
9907 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9908 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9909 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9910 #endif //FREE_USAGE_STATS
9913 void gc_heap::adjust_ephemeral_limits ()
9915 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9916 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9918 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9919 (size_t)ephemeral_low, (size_t)ephemeral_high))
9921 #ifndef MULTIPLE_HEAPS
9922 // This updates the write barrier helpers with the new info.
9923 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9924 #endif // MULTIPLE_HEAPS
9927 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9928 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9932 if (!temp_logfile_name.Get())
9937 char logfile_name[MAX_LONGPATH+1];
9938 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9939 const char* suffix = is_config ? ".config.log" : ".log";
9940 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9941 logFile = fopen(logfile_name, "wb");
9944 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9946 size_t gc_heap::get_segment_size_hard_limit (uint32_t* num_heaps, bool should_adjust_num_heaps)
9948 assert (heap_hard_limit);
9949 size_t aligned_hard_limit = ((heap_hard_limit + min_segment_size_hard_limit - 1) & ~(min_segment_size_hard_limit - 1));
9950 if (should_adjust_num_heaps)
9952 uint32_t max_num_heaps = (uint32_t)(aligned_hard_limit / min_segment_size_hard_limit);
9953 if (*num_heaps > max_num_heaps)
9955 *num_heaps = max_num_heaps;
9959 size_t seg_size = aligned_hard_limit / *num_heaps;
9960 size_t aligned_seg_size = round_up_power2 (seg_size);
9962 assert (g_theGCHeap->IsValidSegmentSize (aligned_seg_size));
9964 size_t seg_size_from_config = (size_t)GCConfig::GetSegmentSize();
9965 if (seg_size_from_config)
9967 size_t aligned_seg_size_config = round_up_power2 (seg_size_from_config);
9969 aligned_seg_size = max (aligned_seg_size, aligned_seg_size_config);
9972 //printf ("limit: %Idmb, aligned: %Idmb, %d heaps, seg size from config: %Idmb, seg size %Idmb",
9973 // (heap_hard_limit / 1024 / 1024),
9974 // (aligned_hard_limit / 1024 / 1024),
9976 // (seg_size_from_config / 1024 / 1024),
9977 // (aligned_seg_size / 1024 / 1024));
9978 return aligned_seg_size;
9981 HRESULT gc_heap::initialize_gc (size_t segment_size,
9983 #ifdef MULTIPLE_HEAPS
9984 ,unsigned number_of_heaps
9985 #endif //MULTIPLE_HEAPS
9989 if (GCConfig::GetLogEnabled())
9991 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9996 // GCLogFileSize in MBs.
9997 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9999 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
10005 gc_log_lock.Initialize();
10006 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
10007 if (!gc_log_buffer)
10013 memset (gc_log_buffer, '*', gc_log_buffer_size);
10015 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
10019 #ifdef GC_CONFIG_DRIVEN
10020 if (GCConfig::GetConfigLogEnabled())
10022 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
10024 if (gc_config_log == NULL)
10027 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
10028 if (!gc_config_log_buffer)
10030 fclose(gc_config_log);
10034 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
10036 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
10037 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
10038 "h#", // heap index
10041 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
10042 "EX", // heap expansion
10043 "NF", // normal fit
10044 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
10047 "PreS", // short object before pinned plug
10048 "PostS", // short object after pinned plug
10049 "Merge", // merged pinned plugs
10050 "Conv", // converted to pinned plug
10051 "Pre", // plug before pinned plug but not after
10052 "Post", // plug after pinned plug but not before
10053 "PrPo", // plug both before and after pinned plug
10054 "PreP", // pre short object padded
10055 "PostP" // post short object padded
10058 #endif //GC_CONFIG_DRIVEN
10061 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
10062 if (logFileName.Get() != nullptr)
10064 GCStatistics::logFileName = _strdup(logFileName.Get());
10065 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
10066 if (!GCStatistics::logFile)
10073 HRESULT hres = S_OK;
10076 hardware_write_watch_api_supported();
10077 #ifdef BACKGROUND_GC
10078 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
10080 gc_can_use_concurrent = true;
10081 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10082 virtual_alloc_hardware_write_watch = true;
10083 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10087 gc_can_use_concurrent = false;
10089 #endif //BACKGROUND_GC
10090 #endif //WRITE_WATCH
10092 #ifdef BACKGROUND_GC
10093 // leave the first page to contain only segment info
10094 // because otherwise we could need to revisit the first page frequently in
10096 segment_info_size = OS_PAGE_SIZE;
10098 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
10099 #endif //BACKGROUND_GC
10101 reserved_memory = 0;
10102 unsigned block_count;
10103 #ifdef MULTIPLE_HEAPS
10104 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
10105 block_count = number_of_heaps;
10106 #else //MULTIPLE_HEAPS
10107 reserved_memory_limit = segment_size + heap_size;
10109 #endif //MULTIPLE_HEAPS
10111 if (heap_hard_limit)
10113 check_commit_cs.Initialize();
10116 if (!reserve_initial_memory(segment_size,heap_size,block_count))
10117 return E_OUTOFMEMORY;
10120 //check if we need to turn on card_bundles.
10121 #ifdef MULTIPLE_HEAPS
10122 // use INT64 arithmetic here because of possible overflow on 32p
10123 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
10125 // use INT64 arithmetic here because of possible overflow on 32p
10126 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
10127 #endif //MULTIPLE_HEAPS
10129 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
10131 settings.card_bundles = TRUE;
10135 settings.card_bundles = FALSE;
10137 #endif //CARD_BUNDLE
10139 settings.first_init();
10141 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
10142 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
10144 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
10147 init_static_data();
10149 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
10151 if (!g_gc_card_table)
10152 return E_OUTOFMEMORY;
10154 gc_started = FALSE;
10156 #ifdef MULTIPLE_HEAPS
10157 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
10159 return E_OUTOFMEMORY;
10162 #pragma warning(push)
10163 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10164 #endif // _PREFAST_
10165 g_promoted = new (nothrow) size_t [number_of_heaps*16];
10166 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
10168 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10169 #endif //MH_SC_MARK
10171 #pragma warning(pop)
10172 #endif // _PREFAST_
10173 if (!g_promoted || !g_bpromoted)
10174 return E_OUTOFMEMORY;
10177 if (!g_mark_stack_busy)
10178 return E_OUTOFMEMORY;
10179 #endif //MH_SC_MARK
10181 if (!create_thread_support (number_of_heaps))
10182 return E_OUTOFMEMORY;
10184 if (!heap_select::init (number_of_heaps))
10185 return E_OUTOFMEMORY;
10187 #endif //MULTIPLE_HEAPS
10189 #ifdef MULTIPLE_HEAPS
10190 yp_spin_count_unit = 32 * number_of_heaps;
10192 yp_spin_count_unit = 32 * g_num_processors;
10193 #endif //MULTIPLE_HEAPS
10195 #if defined(__linux__)
10196 GCToEEInterface::UpdateGCEventStatus(static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Default)),
10197 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Default)),
10198 static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Private)),
10199 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Private)));
10200 #endif // __linux__
10202 if (!init_semi_shared())
10210 //Initializes PER_HEAP_ISOLATED data members.
10212 gc_heap::init_semi_shared()
10216 // This is used for heap expansion - it's to fix exactly the start for gen 0
10217 // through (max_generation-1). When we expand the heap we allocate all these
10218 // gen starts at the beginning of the new ephemeral seg.
10219 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10222 #ifdef MULTIPLE_HEAPS
10223 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10224 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10226 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10227 #ifdef PARALLEL_MARK_LIST_SORT
10228 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10229 if (!g_mark_list_copy)
10233 #endif //PARALLEL_MARK_LIST_SORT
10235 #else //MULTIPLE_HEAPS
10237 mark_list_size = max (8192, soh_segment_size/(64*32));
10238 g_mark_list = make_mark_list (mark_list_size);
10240 #endif //MULTIPLE_HEAPS
10242 dprintf (3, ("mark_list_size: %d", mark_list_size));
10250 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10251 if (!seg_mapping_table_init())
10253 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10255 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10256 seg_table = sorted_table::make_sorted_table();
10260 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10262 segment_standby_list = 0;
10264 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10268 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10273 fgn_maxgen_percent = 0;
10274 fgn_loh_percent = 0;
10275 full_gc_approach_event_set = false;
10277 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10280 should_expand_in_full_gc = FALSE;
10282 #ifdef FEATURE_LOH_COMPACTION
10283 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10284 loh_compaction_mode = loh_compaction_default;
10285 #endif //FEATURE_LOH_COMPACTION
10287 loh_size_threshold = (size_t)GCConfig::GetLOHThreshold();
10288 assert (loh_size_threshold >= LARGE_OBJECT_SIZE);
10290 #ifdef BACKGROUND_GC
10291 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10292 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10293 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10296 int number_bgc_threads = 1;
10297 #ifdef MULTIPLE_HEAPS
10298 number_bgc_threads = n_heaps;
10299 #endif //MULTIPLE_HEAPS
10300 if (!create_bgc_threads_support (number_bgc_threads))
10305 #endif //BACKGROUND_GC
10307 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10309 #ifdef GC_CONFIG_DRIVEN
10310 compact_or_sweep_gcs[0] = 0;
10311 compact_or_sweep_gcs[1] = 0;
10312 #endif //GC_CONFIG_DRIVEN
10315 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10316 #endif //SHORT_PLUGS
10324 if (full_gc_approach_event.IsValid())
10326 full_gc_approach_event.CloseEvent();
10328 if (full_gc_end_event.IsValid())
10330 full_gc_end_event.CloseEvent();
10337 gc_heap* gc_heap::make_gc_heap (
10338 #ifdef MULTIPLE_HEAPS
10341 #endif //MULTIPLE_HEAPS
10346 #ifdef MULTIPLE_HEAPS
10347 res = new (nothrow) gc_heap;
10351 res->vm_heap = vm_hp;
10352 res->alloc_context_count = 0;
10355 #ifdef PARALLEL_MARK_LIST_SORT
10356 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10357 if (!res->mark_list_piece_start)
10361 #pragma warning(push)
10362 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10363 #endif // _PREFAST_
10364 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10366 #pragma warning(pop)
10367 #endif // _PREFAST_
10369 if (!res->mark_list_piece_end)
10371 #endif //PARALLEL_MARK_LIST_SORT
10375 #endif //MULTIPLE_HEAPS
10377 if (res->init_gc_heap (
10378 #ifdef MULTIPLE_HEAPS
10380 #else //MULTIPLE_HEAPS
10382 #endif //MULTIPLE_HEAPS
10388 #ifdef MULTIPLE_HEAPS
10391 return (gc_heap*)1;
10392 #endif //MULTIPLE_HEAPS
10396 gc_heap::wait_for_gc_done(int32_t timeOut)
10398 bool cooperative_mode = enable_preemptive ();
10400 uint32_t dwWaitResult = NOERROR;
10402 gc_heap* wait_heap = NULL;
10403 while (gc_heap::gc_started)
10405 #ifdef MULTIPLE_HEAPS
10406 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10407 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10408 #endif // MULTIPLE_HEAPS
10411 PREFIX_ASSUME(wait_heap != NULL);
10412 #endif // _PREFAST_
10414 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10416 disable_preemptive (cooperative_mode);
10418 return dwWaitResult;
10422 gc_heap::set_gc_done()
10424 enter_gc_done_event_lock();
10425 if (!gc_done_event_set)
10427 gc_done_event_set = true;
10428 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10429 gc_done_event.Set();
10431 exit_gc_done_event_lock();
10435 gc_heap::reset_gc_done()
10437 enter_gc_done_event_lock();
10438 if (gc_done_event_set)
10440 gc_done_event_set = false;
10441 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10442 gc_done_event.Reset();
10444 exit_gc_done_event_lock();
10448 gc_heap::enter_gc_done_event_lock()
10450 uint32_t dwSwitchCount = 0;
10453 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10455 while (gc_done_event_lock >= 0)
10457 if (g_num_processors > 1)
10459 int spin_count = yp_spin_count_unit;
10460 for (int j = 0; j < spin_count; j++)
10462 if (gc_done_event_lock < 0)
10464 YieldProcessor(); // indicate to the processor that we are spinning
10466 if (gc_done_event_lock >= 0)
10467 GCToOSInterface::YieldThread(++dwSwitchCount);
10470 GCToOSInterface::YieldThread(++dwSwitchCount);
10477 gc_heap::exit_gc_done_event_lock()
10479 gc_done_event_lock = -1;
10482 #ifndef MULTIPLE_HEAPS
10484 #ifdef RECORD_LOH_STATE
10485 int gc_heap::loh_state_index = 0;
10486 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10487 #endif //RECORD_LOH_STATE
10489 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10490 VOLATILE(bool) gc_heap::gc_done_event_set;
10491 GCEvent gc_heap::gc_done_event;
10492 #endif //!MULTIPLE_HEAPS
10493 VOLATILE(bool) gc_heap::internal_gc_done;
10495 void gc_heap::add_saved_spinlock_info (
10497 msl_enter_state enter_state,
10498 msl_take_state take_state)
10501 #ifdef SPINLOCK_HISTORY
10502 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10504 current->enter_state = enter_state;
10505 current->take_state = take_state;
10506 current->thread_id.SetToCurrentThread();
10507 current->loh_p = loh_p;
10508 dprintf (SPINLOCK_LOG, ("[%d]%s %s %s",
10510 (loh_p ? "loh" : "soh"),
10511 ((enter_state == me_acquire) ? "E" : "L"),
10512 msl_take_state_str[take_state]));
10514 spinlock_info_index++;
10516 assert (spinlock_info_index <= max_saved_spinlock_info);
10518 if (spinlock_info_index >= max_saved_spinlock_info)
10520 spinlock_info_index = 0;
10523 MAYBE_UNUSED_VAR(enter_state);
10524 MAYBE_UNUSED_VAR(take_state);
10525 #endif //SPINLOCK_HISTORY
10529 gc_heap::init_gc_heap (int h_number)
10531 #ifdef MULTIPLE_HEAPS
10535 allocated_since_last_gc = 0;
10537 #ifdef SPINLOCK_HISTORY
10538 spinlock_info_index = 0;
10539 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10540 #endif //SPINLOCK_HISTORY
10542 // initialize per heap members.
10543 ephemeral_low = (uint8_t*)1;
10545 ephemeral_high = MAX_PTR;
10547 ephemeral_heap_segment = 0;
10549 freeable_large_heap_segment = 0;
10551 condemned_generation_num = 0;
10553 blocking_collection = FALSE;
10555 generation_skip_ratio = 100;
10557 mark_stack_tos = 0;
10559 mark_stack_bos = 0;
10561 mark_stack_array_length = 0;
10563 mark_stack_array = 0;
10565 #if defined (_DEBUG) && defined (VERIFY_HEAP)
10566 verify_pinned_queue_p = FALSE;
10567 #endif // _DEBUG && VERIFY_HEAP
10569 loh_pinned_queue_tos = 0;
10571 loh_pinned_queue_bos = 0;
10573 loh_pinned_queue_length = 0;
10575 loh_pinned_queue_decay = LOH_PIN_DECAY;
10577 loh_pinned_queue = 0;
10579 min_overflow_address = MAX_PTR;
10581 max_overflow_address = 0;
10583 gen0_bricks_cleared = FALSE;
10585 gen0_must_clear_bricks = 0;
10587 allocation_quantum = CLR_SIZE;
10589 more_space_lock_soh = gc_lock;
10591 more_space_lock_loh = gc_lock;
10593 ro_segments_in_range = FALSE;
10595 loh_alloc_since_cg = 0;
10597 new_heap_segment = NULL;
10599 gen0_allocated_after_gc_p = false;
10601 #ifdef RECORD_LOH_STATE
10602 loh_state_index = 0;
10603 #endif //RECORD_LOH_STATE
10604 #endif //MULTIPLE_HEAPS
10606 #ifdef MULTIPLE_HEAPS
10607 if (h_number > n_heaps)
10609 assert (!"Number of heaps exceeded");
10613 heap_number = h_number;
10614 #endif //MULTIPLE_HEAPS
10616 memset (&oom_info, 0, sizeof (oom_info));
10617 memset (&fgm_result, 0, sizeof (fgm_result));
10618 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10622 gc_done_event_lock = -1;
10623 gc_done_event_set = false;
10625 #ifndef SEG_MAPPING_TABLE
10626 if (!gc_heap::seg_table->ensure_space_for_insert ())
10630 #endif //!SEG_MAPPING_TABLE
10632 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10636 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10637 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10638 gc_etw_segment_small_object_heap);
10640 #ifdef SEG_MAPPING_TABLE
10641 seg_mapping_table_add_segment (seg, __this);
10642 #else //SEG_MAPPING_TABLE
10643 seg_table->insert ((uint8_t*)seg, sdelta);
10644 #endif //SEG_MAPPING_TABLE
10646 #ifdef MULTIPLE_HEAPS
10647 heap_segment_heap (seg) = this;
10648 #endif //MULTIPLE_HEAPS
10650 /* todo: Need a global lock for this */
10651 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10652 own_card_table (ct);
10653 card_table = translate_card_table (ct);
10654 /* End of global lock */
10656 brick_table = card_table_brick_table (ct);
10657 highest_address = card_table_highest_address (ct);
10658 lowest_address = card_table_lowest_address (ct);
10661 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10662 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10663 card_table_card_bundle_table (ct));
10664 #endif //CARD_BUNDLE
10667 if (gc_can_use_concurrent)
10668 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10671 #endif //MARK_ARRAY
10673 uint8_t* start = heap_segment_mem (seg);
10675 for (int i = 0; i < 1 + max_generation; i++)
10677 make_generation (generation_table [ (max_generation - i) ],
10679 generation_table [(max_generation - i)].gen_num = max_generation - i;
10680 start += Align (min_obj_size);
10683 heap_segment_allocated (seg) = start;
10684 alloc_allocated = start;
10685 heap_segment_used (seg) = start - plug_skew;
10687 ephemeral_heap_segment = seg;
10689 #ifndef SEG_MAPPING_TABLE
10690 if (!gc_heap::seg_table->ensure_space_for_insert ())
10694 #endif //!SEG_MAPPING_TABLE
10695 //Create the large segment generation
10696 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10699 lseg->flags |= heap_segment_flags_loh;
10701 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10702 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10703 gc_etw_segment_large_object_heap);
10705 #ifdef SEG_MAPPING_TABLE
10706 seg_mapping_table_add_segment (lseg, __this);
10707 #else //SEG_MAPPING_TABLE
10708 seg_table->insert ((uint8_t*)lseg, sdelta);
10709 #endif //SEG_MAPPING_TABLE
10711 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10712 //assign the alloc_list for the large generation
10713 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10714 generation_table [max_generation+1].gen_num = max_generation+1;
10715 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10716 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10717 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10719 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10721 generation* gen = generation_of (gen_num);
10722 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10725 #ifdef MULTIPLE_HEAPS
10726 heap_segment_heap (lseg) = this;
10728 //initialize the alloc context heap
10729 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10731 //initialize the alloc context heap
10732 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10734 #endif //MULTIPLE_HEAPS
10736 //Do this only once
10737 #ifdef MULTIPLE_HEAPS
10739 #endif //MULTIPLE_HEAPS
10741 #ifndef INTERIOR_POINTERS
10742 //set the brick_table for large objects
10743 //but default value is clearded
10744 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10745 // (uint8_t*)heap_segment_reserved (lseg));
10747 #else //INTERIOR_POINTERS
10749 //Because of the interior pointer business, we have to clear
10750 //the whole brick table
10751 //but the default value is cleared
10752 // clear_brick_table (lowest_address, highest_address);
10753 #endif //INTERIOR_POINTERS
10756 if (!init_dynamic_data())
10761 etw_allocation_running_amount[0] = 0;
10762 etw_allocation_running_amount[1] = 0;
10764 //needs to be done after the dynamic data has been initialized
10765 #ifndef MULTIPLE_HEAPS
10766 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10767 #endif //!MULTIPLE_HEAPS
10769 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10771 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10775 make_mark_stack(arr);
10777 #ifdef BACKGROUND_GC
10778 freeable_small_heap_segment = 0;
10779 gchist_index_per_heap = 0;
10780 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10784 make_background_mark_stack (b_arr);
10785 #endif //BACKGROUND_GC
10787 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10788 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10789 if (heap_number == 0)
10791 stomp_write_barrier_initialize(
10792 #ifdef MULTIPLE_HEAPS
10793 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10795 ephemeral_low, ephemeral_high
10796 #endif //!MULTIPLE_HEAPS
10801 // why would we clear the mark array for this page? it should be cleared..
10802 // clear the first committed page
10803 //if(gc_can_use_concurrent)
10805 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10807 #endif //MARK_ARRAY
10809 #ifdef MULTIPLE_HEAPS
10810 //register the heap in the heaps array
10812 if (!create_gc_thread ())
10815 g_heaps [heap_number] = this;
10817 #endif //MULTIPLE_HEAPS
10819 #ifdef FEATURE_PREMORTEM_FINALIZATION
10820 HRESULT hr = AllocateCFinalize(&finalize_queue);
10823 #endif // FEATURE_PREMORTEM_FINALIZATION
10825 max_free_space_items = MAX_NUM_FREE_SPACES;
10827 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10834 if (!bestfit_seg->alloc())
10839 last_gc_before_oom = FALSE;
10841 sufficient_gen0_space_p = FALSE;
10843 #ifdef MULTIPLE_HEAPS
10845 #ifdef HEAP_ANALYZE
10847 heap_analyze_success = TRUE;
10849 internal_root_array = 0;
10851 internal_root_array_index = 0;
10853 internal_root_array_length = initial_internal_roots;
10857 current_obj_size = 0;
10859 #endif //HEAP_ANALYZE
10861 #endif // MULTIPLE_HEAPS
10863 #ifdef BACKGROUND_GC
10864 bgc_thread_id.Clear();
10866 if (!create_bgc_thread_support())
10871 bgc_alloc_lock = new (nothrow) exclusive_sync;
10872 if (!bgc_alloc_lock)
10877 bgc_alloc_lock->init();
10881 if (!recursive_gc_sync::init())
10885 bgc_thread_running = 0;
10887 bgc_threads_timeout_cs.Initialize();
10888 expanded_in_fgc = 0;
10889 current_bgc_state = bgc_not_in_process;
10890 background_soh_alloc_count = 0;
10891 background_loh_alloc_count = 0;
10892 bgc_overflow_count = 0;
10893 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10894 #endif //BACKGROUND_GC
10896 #ifdef GC_CONFIG_DRIVEN
10897 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10898 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10899 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10900 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10901 #endif //GC_CONFIG_DRIVEN
10907 gc_heap::destroy_semi_shared()
10909 //TODO: will need to move this to per heap
10910 //#ifdef BACKGROUND_GC
10911 // if (c_mark_list)
10912 // delete c_mark_list;
10913 //#endif //BACKGROUND_GC
10917 delete g_mark_list;
10920 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10921 if (seg_mapping_table)
10922 delete seg_mapping_table;
10923 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10925 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10926 //destroy the segment map
10927 seg_table->delete_sorted_table();
10928 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10932 gc_heap::self_destroy()
10934 #ifdef BACKGROUND_GC
10936 #endif //BACKGROUND_GC
10938 if (gc_done_event.IsValid())
10940 gc_done_event.CloseEvent();
10943 // destroy every segment.
10944 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10946 PREFIX_ASSUME(seg != NULL);
10948 heap_segment* next_seg;
10951 next_seg = heap_segment_next_rw (seg);
10952 delete_heap_segment (seg);
10956 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10958 PREFIX_ASSUME(seg != NULL);
10962 next_seg = heap_segment_next_rw (seg);
10963 delete_heap_segment (seg);
10967 // get rid of the card table
10968 release_card_table (card_table);
10970 // destroy the mark stack
10971 delete mark_stack_array;
10973 #ifdef FEATURE_PREMORTEM_FINALIZATION
10974 if (finalize_queue)
10975 delete finalize_queue;
10976 #endif // FEATURE_PREMORTEM_FINALIZATION
10980 gc_heap::destroy_gc_heap(gc_heap* heap)
10982 heap->self_destroy();
10986 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10987 // the finalizer queue has been drained.
10988 void gc_heap::shutdown_gc()
10990 destroy_semi_shared();
10992 #ifdef MULTIPLE_HEAPS
10993 //delete the heaps array
10995 destroy_thread_support();
10997 #endif //MULTIPLE_HEAPS
10998 //destroy seg_manager
11000 destroy_initial_memory();
11002 GCToOSInterface::Shutdown();
11006 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11007 uint8_t* old_loc, int use_padding)
11009 BOOL already_padded = FALSE;
11011 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
11013 alloc_pointer = alloc_pointer + Align (min_obj_size);
11014 already_padded = TRUE;
11016 #endif //SHORT_PLUGS
11018 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
11019 size = size + switch_alignment_size (already_padded);
11021 #ifdef FEATURE_STRUCTALIGN
11022 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
11023 #endif // FEATURE_STRUCTALIGN
11025 // in allocate_in_condemned_generation we can have this when we
11026 // set the alloc_limit to plan_allocated which could be less than
11028 if (alloc_limit < alloc_pointer)
11035 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
11037 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
11038 #else //SHORT_PLUGS
11039 ||((alloc_pointer + size) == alloc_limit)
11040 #endif //SHORT_PLUGS
11045 assert (size == Align (min_obj_size));
11046 return ((size_t)(alloc_limit - alloc_pointer) >= size);
11051 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11054 // We could have run into cases where this is true when alloc_allocated is the
11055 // the same as the seg committed.
11056 if (alloc_limit < alloc_pointer)
11061 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
11064 // Grow by committing more pages
11065 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* hard_limit_exceeded_p)
11067 assert (high_address <= heap_segment_reserved (seg));
11069 if (hard_limit_exceeded_p)
11070 *hard_limit_exceeded_p = false;
11072 //return 0 if we are at the end of the segment.
11073 if (align_on_page (high_address) > heap_segment_reserved (seg))
11076 if (high_address <= heap_segment_committed (seg))
11079 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
11080 c_size = max (c_size, commit_min_th);
11081 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
11086 STRESS_LOG2(LF_GC, LL_INFO10000,
11087 "Growing heap_segment: %Ix high address: %Ix\n",
11088 (size_t)seg, (size_t)high_address);
11090 bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_number, hard_limit_exceeded_p);
11094 #ifndef BACKGROUND_GC
11095 clear_mark_array (heap_segment_committed (seg),
11096 heap_segment_committed (seg)+c_size, TRUE);
11097 #endif //BACKGROUND_GC
11098 #endif //MARK_ARRAY
11099 heap_segment_committed (seg) += c_size;
11101 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
11102 (size_t)heap_segment_committed (seg));
11104 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
11105 assert (high_address <= heap_segment_committed (seg));
11112 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
11115 if ((old_loc != 0) && pad_front_p)
11117 allocated = allocated + Align (min_obj_size);
11119 #endif //SHORT_PLUGS
11121 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
11122 size = size + switch_alignment_size (FALSE);
11123 #ifdef FEATURE_STRUCTALIGN
11124 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
11125 return grow_heap_segment (seg, allocated + pad + size);
11126 #else // FEATURE_STRUCTALIGN
11127 return grow_heap_segment (seg, allocated + size);
11128 #endif // FEATURE_STRUCTALIGN
11131 //used only in older generation allocation (i.e during gc).
11132 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
11135 UNREFERENCED_PARAMETER(gennum);
11136 dprintf (3, ("gc Expanding segment allocation"));
11137 heap_segment* seg = generation_allocation_segment (gen);
11138 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
11140 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
11142 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
11143 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
11144 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
11148 uint8_t* hole = generation_allocation_pointer (gen);
11149 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
11153 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
11154 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
11155 if (size >= Align (min_free_list))
11157 if (allocated_size < min_free_list)
11159 if (size >= (Align (min_free_list) + Align (min_obj_size)))
11161 //split hole into min obj + threadable free item
11162 make_unused_array (hole, min_obj_size);
11163 generation_free_obj_space (gen) += Align (min_obj_size);
11164 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
11165 generation_free_list_space (gen) += size - Align (min_obj_size);
11166 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
11167 size - Align (min_obj_size));
11168 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
11172 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
11173 make_unused_array (hole, size);
11174 generation_free_obj_space (gen) += size;
11179 dprintf (3, ("threading hole in front of free list"));
11180 make_unused_array (hole, size);
11181 generation_free_list_space (gen) += size;
11182 generation_allocator(gen)->thread_item_front (hole, size);
11183 add_gen_free (gen->gen_num, size);
11188 make_unused_array (hole, size);
11189 generation_free_obj_space (gen) += size;
11193 generation_allocation_pointer (gen) = start;
11194 generation_allocation_context_start_region (gen) = start;
11196 generation_allocation_limit (gen) = (start + limit_size);
11199 void verify_mem_cleared (uint8_t* start, size_t size)
11201 if (!Aligned (size))
11206 PTR_PTR curr_ptr = (PTR_PTR) start;
11207 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11209 if (*(curr_ptr++) != 0)
11216 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11217 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11219 size_t start_mark_bit = mark_bit_of (start);
11220 size_t end_mark_bit = mark_bit_of (end);
11221 unsigned int startbit = mark_bit_bit (start_mark_bit);
11222 unsigned int endbit = mark_bit_bit (end_mark_bit);
11223 size_t startwrd = mark_bit_word (start_mark_bit);
11224 size_t endwrd = mark_bit_word (end_mark_bit);
11226 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11227 (size_t)start, (size_t)start_mark_bit,
11228 (size_t)end, (size_t)end_mark_bit));
11230 unsigned int firstwrd = ~(lowbits (~0, startbit));
11231 unsigned int lastwrd = ~(highbits (~0, endbit));
11233 if (startwrd == endwrd)
11235 unsigned int wrd = firstwrd & lastwrd;
11236 mark_array[startwrd] |= wrd;
11240 // set the first mark word.
11243 mark_array[startwrd] |= firstwrd;
11247 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11249 mark_array[wrdtmp] = ~(unsigned int)0;
11252 // set the last mark word.
11255 mark_array[endwrd] |= lastwrd;
11259 // makes sure that the mark array bits between start and end are 0.
11260 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11262 size_t start_mark_bit = mark_bit_of (start);
11263 size_t end_mark_bit = mark_bit_of (end);
11264 unsigned int startbit = mark_bit_bit (start_mark_bit);
11265 unsigned int endbit = mark_bit_bit (end_mark_bit);
11266 size_t startwrd = mark_bit_word (start_mark_bit);
11267 size_t endwrd = mark_bit_word (end_mark_bit);
11269 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11270 // (size_t)start, (size_t)start_mark_bit,
11271 // (size_t)end, (size_t)end_mark_bit));
11273 unsigned int firstwrd = ~(lowbits (~0, startbit));
11274 unsigned int lastwrd = ~(highbits (~0, endbit));
11276 if (startwrd == endwrd)
11278 unsigned int wrd = firstwrd & lastwrd;
11279 if (mark_array[startwrd] & wrd)
11281 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11283 mark_array [startwrd], mark_word_address (startwrd)));
11289 // set the first mark word.
11292 if (mark_array[startwrd] & firstwrd)
11294 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11295 firstwrd, startwrd,
11296 mark_array [startwrd], mark_word_address (startwrd)));
11303 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11305 if (mark_array[wrdtmp])
11307 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11309 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11314 // set the last mark word.
11317 if (mark_array[endwrd] & lastwrd)
11319 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11321 mark_array [lastwrd], mark_word_address (lastwrd)));
11326 #endif //VERIFY_HEAP && BACKGROUND_GC
11328 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11330 assert (num_b < MAX_BUCKET_COUNT);
11331 num_buckets = num_b;
11332 frst_bucket_size = fbs;
11336 alloc_list& allocator::alloc_list_of (unsigned int bn)
11338 assert (bn < num_buckets);
11340 return first_bucket;
11342 return buckets [bn-1];
11345 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11347 assert (bn < num_buckets);
11349 return first_bucket.alloc_list_damage_count();
11351 return buckets [bn-1].alloc_list_damage_count();
11354 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11356 //unlink the free_item
11357 alloc_list* al = &alloc_list_of (bn);
11360 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11362 assert (item == free_list_slot (prev_item));
11363 free_list_undo (prev_item) = item;
11364 alloc_list_damage_count_of (bn)++;
11366 free_list_slot (prev_item) = free_list_slot(item);
11370 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11372 if (al->alloc_list_tail() == item)
11374 al->alloc_list_tail() = prev_item;
11378 void allocator::clear()
11380 for (unsigned int i = 0; i < num_buckets; i++)
11382 alloc_list_head_of (i) = 0;
11383 alloc_list_tail_of (i) = 0;
11387 //always thread to the end.
11388 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11390 free_list_slot (item) = 0;
11391 free_list_undo (item) = UNDO_EMPTY;
11392 assert (item != head);
11398 //TODO: This shouldn't happen anymore - verify that's the case.
11399 //the following is necessary because the last free element
11400 //may have been truncated, and tail isn't updated.
11401 else if (free_list_slot (head) == 0)
11403 free_list_slot (head) = item;
11407 assert (item != tail);
11408 assert (free_list_slot(tail) == 0);
11409 free_list_slot (tail) = item;
11414 void allocator::thread_item (uint8_t* item, size_t size)
11416 size_t sz = frst_bucket_size;
11417 unsigned int a_l_number = 0;
11419 for (; a_l_number < (num_buckets-1); a_l_number++)
11427 alloc_list* al = &alloc_list_of (a_l_number);
11428 thread_free_item (item,
11429 al->alloc_list_head(),
11430 al->alloc_list_tail());
11433 void allocator::thread_item_front (uint8_t* item, size_t size)
11435 //find right free list
11436 size_t sz = frst_bucket_size;
11437 unsigned int a_l_number = 0;
11438 for (; a_l_number < (num_buckets-1); a_l_number++)
11446 alloc_list* al = &alloc_list_of (a_l_number);
11447 free_list_slot (item) = al->alloc_list_head();
11448 free_list_undo (item) = UNDO_EMPTY;
11450 if (al->alloc_list_tail() == 0)
11452 al->alloc_list_tail() = al->alloc_list_head();
11454 al->alloc_list_head() = item;
11455 if (al->alloc_list_tail() == 0)
11457 al->alloc_list_tail() = item;
11461 void allocator::copy_to_alloc_list (alloc_list* toalist)
11463 for (unsigned int i = 0; i < num_buckets; i++)
11465 toalist [i] = alloc_list_of (i);
11466 #ifdef FL_VERIFICATION
11467 uint8_t* free_item = alloc_list_head_of (i);
11472 free_item = free_list_slot (free_item);
11475 toalist[i].item_count = count;
11476 #endif //FL_VERIFICATION
11480 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11482 BOOL repair_list = !discard_if_no_fit_p ();
11483 for (unsigned int i = 0; i < num_buckets; i++)
11485 size_t count = alloc_list_damage_count_of (i);
11486 alloc_list_of (i) = fromalist [i];
11487 assert (alloc_list_damage_count_of (i) == 0);
11491 //repair the the list
11492 //new items may have been added during the plan phase
11493 //items may have been unlinked.
11494 uint8_t* free_item = alloc_list_head_of (i);
11495 while (free_item && count)
11497 assert (((CObjectHeader*)free_item)->IsFree());
11498 if ((free_list_undo (free_item) != UNDO_EMPTY))
11501 free_list_slot (free_item) = free_list_undo (free_item);
11502 free_list_undo (free_item) = UNDO_EMPTY;
11505 free_item = free_list_slot (free_item);
11508 #ifdef FL_VERIFICATION
11509 free_item = alloc_list_head_of (i);
11510 size_t item_count = 0;
11514 free_item = free_list_slot (free_item);
11517 assert (item_count == alloc_list_of (i).item_count);
11518 #endif //FL_VERIFICATION
11521 uint8_t* tail_item = alloc_list_tail_of (i);
11522 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11527 void allocator::commit_alloc_list_changes()
11529 BOOL repair_list = !discard_if_no_fit_p ();
11532 for (unsigned int i = 0; i < num_buckets; i++)
11534 //remove the undo info from list.
11535 uint8_t* free_item = alloc_list_head_of (i);
11536 size_t count = alloc_list_damage_count_of (i);
11537 while (free_item && count)
11539 assert (((CObjectHeader*)free_item)->IsFree());
11541 if (free_list_undo (free_item) != UNDO_EMPTY)
11543 free_list_undo (free_item) = UNDO_EMPTY;
11547 free_item = free_list_slot (free_item);
11550 alloc_list_damage_count_of (i) = 0;
11555 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11556 alloc_context* acontext, heap_segment* seg,
11557 int align_const, int gen_number)
11559 bool loh_p = (gen_number > 0);
11560 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
11562 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11566 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11569 #ifdef MULTIPLE_HEAPS
11570 if (gen_number == 0)
11572 if (!gen0_allocated_after_gc_p)
11574 gen0_allocated_after_gc_p = true;
11577 #endif //MULTIPLE_HEAPS
11579 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11580 (size_t)start + limit_size - aligned_min_obj_size));
11582 if ((acontext->alloc_limit != start) &&
11583 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11585 uint8_t* hole = acontext->alloc_ptr;
11588 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11589 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11590 // when we are finishing an allocation from a free list
11591 // we know that the free area was Align(min_obj_size) larger
11592 acontext->alloc_bytes -= size;
11593 size_t free_obj_size = size + aligned_min_obj_size;
11594 make_unused_array (hole, free_obj_size);
11595 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11597 acontext->alloc_ptr = start;
11601 if (gen_number == 0)
11603 size_t pad_size = Align (min_obj_size, align_const);
11604 make_unused_array (acontext->alloc_ptr, pad_size);
11605 dprintf (3, ("contigous ac: making min obj gap %Ix->%Ix(%Id)",
11606 acontext->alloc_ptr, (acontext->alloc_ptr + pad_size), pad_size));
11607 acontext->alloc_ptr += pad_size;
11610 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11611 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11613 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11614 if (g_fEnableAppDomainMonitoring)
11616 GCToEEInterface::RecordAllocatedBytesForHeap(limit_size, heap_number);
11618 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11620 uint8_t* saved_used = 0;
11624 saved_used = heap_segment_used (seg);
11627 if (seg == ephemeral_heap_segment)
11629 //Sometimes the allocated size is advanced without clearing the
11630 //memory. Let's catch up here
11631 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11634 #ifndef BACKGROUND_GC
11635 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11636 #endif //BACKGROUND_GC
11637 #endif //MARK_ARRAY
11638 heap_segment_used (seg) = alloc_allocated - plug_skew;
11641 #ifdef BACKGROUND_GC
11644 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11645 #ifdef FEATURE_LOH_COMPACTION
11646 old_allocated -= Align (loh_padding_obj_size, align_const);
11647 #endif //FEATURE_LOH_COMPACTION
11649 assert (heap_segment_used (seg) >= old_allocated);
11651 #endif //BACKGROUND_GC
11653 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11655 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11656 leave_spin_lock (msl);
11657 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11658 memclr (start - plug_skew, limit_size);
11662 uint8_t* used = heap_segment_used (seg);
11663 heap_segment_used (seg) = start + limit_size - plug_skew;
11665 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11666 leave_spin_lock (msl);
11668 if ((start - plug_skew) < used)
11670 if (used != saved_used)
11675 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11676 (start - plug_skew), (plug_skew + used - start)));
11677 memclr (start - plug_skew, used - (start - plug_skew));
11681 //this portion can be done after we release the lock
11682 if (seg == ephemeral_heap_segment)
11684 #ifdef FFIND_OBJECT
11685 if (gen0_must_clear_bricks > 0)
11687 //set the brick table to speed up find_object
11688 size_t b = brick_of (acontext->alloc_ptr);
11689 set_brick (b, acontext->alloc_ptr - brick_address (b));
11691 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11692 b, brick_of (align_on_brick (start + limit_size))));
11693 volatile short* x = &brick_table [b];
11694 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11696 for (;x < end_x;x++)
11700 #endif //FFIND_OBJECT
11702 gen0_bricks_cleared = FALSE;
11706 // verifying the memory is completely cleared.
11707 //verify_mem_cleared (start - plug_skew, limit_size);
11710 size_t gc_heap::new_allocation_limit (size_t size, size_t physical_limit, int gen_number)
11712 dynamic_data* dd = dynamic_data_of (gen_number);
11713 ptrdiff_t new_alloc = dd_new_allocation (dd);
11714 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
11715 get_alignment_constant (!(gen_number == (max_generation+1)))));
11717 ptrdiff_t logical_limit = max (new_alloc, (ptrdiff_t)size);
11718 size_t limit = min (logical_limit, (ptrdiff_t)physical_limit);
11719 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
11720 dd_new_allocation (dd) = (new_alloc - limit);
11724 size_t gc_heap::limit_from_size (size_t size, size_t physical_limit, int gen_number,
11727 size_t padded_size = size + Align (min_obj_size, align_const);
11728 // for LOH this is not true...we could select a physical_limit that's exactly the same
11730 assert ((gen_number != 0) || (physical_limit >= padded_size));
11731 size_t min_size_to_allocate = ((gen_number == 0) ? allocation_quantum : 0);
11733 // For SOH if the size asked for is very small, we want to allocate more than
11734 // just what's asked for if possible.
11735 size_t desired_size_to_allocate = max (padded_size, min_size_to_allocate);
11736 size_t new_physical_limit = min (physical_limit, desired_size_to_allocate);
11738 size_t new_limit = new_allocation_limit (padded_size,
11739 new_physical_limit,
11741 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11742 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11746 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11747 uint8_t* allocated, uint8_t* reserved)
11749 UNREFERENCED_PARAMETER(heap_num);
11751 if (reason == oom_budget)
11753 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11756 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11758 // This means during the last GC we needed to reserve and/or commit more memory
11759 // but we couldn't. We proceeded with the GC and ended up not having enough
11760 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11761 // probably made a mistake and didn't expand the heap when we should have.
11762 reason = oom_low_mem;
11765 oom_info.reason = reason;
11766 oom_info.allocated = allocated;
11767 oom_info.reserved = reserved;
11768 oom_info.alloc_size = alloc_size;
11769 oom_info.gc_index = settings.gc_index;
11770 oom_info.fgm = fgm_result.fgm;
11771 oom_info.size = fgm_result.size;
11772 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11773 oom_info.loh_p = fgm_result.loh_p;
11775 fgm_result.fgm = fgm_no_failure;
11777 // Break early - before the more_space_lock is release so no other threads
11778 // could have allocated on the same heap when OOM happened.
11779 if (GCConfig::GetBreakOnOOM())
11781 GCToOSInterface::DebugBreak();
11785 #ifdef BACKGROUND_GC
11786 BOOL gc_heap::background_allowed_p()
11788 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11790 #endif //BACKGROUND_GC
11792 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11794 BOOL should_notify = FALSE;
11795 // if we detect full gc because of the allocation budget specified this is TRUE;
11796 // it's FALSE if it's due to other factors.
11797 BOOL alloc_factor = TRUE;
11800 int n_initial = gen_num;
11801 BOOL local_blocking_collection = FALSE;
11802 BOOL local_elevation_requested = FALSE;
11803 int new_alloc_remain_percent = 0;
11805 if (full_gc_approach_event_set)
11810 if (gen_num != (max_generation + 1))
11812 gen_num = max_generation;
11815 dynamic_data* dd_full = dynamic_data_of (gen_num);
11816 ptrdiff_t new_alloc_remain = 0;
11817 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11819 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11821 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11822 heap_number, gen_index,
11823 dd_new_allocation (dynamic_data_of (gen_index)),
11824 dd_desired_allocation (dynamic_data_of (gen_index))));
11827 // For small object allocations we only check every fgn_check_quantum bytes.
11828 if (n_initial == 0)
11830 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11831 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11832 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11833 (dd_new_allocation (dd_0) >= 0))
11839 fgn_last_alloc = dd_new_allocation (dd_0);
11840 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11843 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11848 for (i = n+1; i <= max_generation; i++)
11850 if (get_new_allocation (i) <= 0)
11852 n = min (i, max_generation);
11858 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11859 if (gen_num == max_generation)
11861 // If it's small object heap we should first see if we will even be looking at gen2 budget
11862 // in the next GC or not. If not we should go directly to checking other factors.
11863 if (n < (max_generation - 1))
11865 goto check_other_factors;
11869 new_alloc_remain = dd_new_allocation (dd_full) - size;
11871 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11873 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11874 gen_num, pct, new_alloc_remain_percent));
11876 if (new_alloc_remain_percent <= (int)pct)
11878 #ifdef BACKGROUND_GC
11879 // If background GC is enabled, we still want to check whether this will
11880 // be a blocking GC or not because we only want to notify when it's a
11881 // blocking full GC.
11882 if (background_allowed_p())
11884 goto check_other_factors;
11886 #endif //BACKGROUND_GC
11888 should_notify = TRUE;
11892 check_other_factors:
11894 dprintf (2, ("FGC: checking other factors"));
11895 n = generation_to_condemn (n,
11896 &local_blocking_collection,
11897 &local_elevation_requested,
11900 if (local_elevation_requested && (n == max_generation))
11902 if (settings.should_lock_elevation)
11904 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11905 if (local_elevation_locked_count != 6)
11907 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11908 local_elevation_locked_count));
11909 n = max_generation - 1;
11914 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11916 #ifdef BACKGROUND_GC
11917 // When background GC is enabled it decreases the accuracy of our predictability -
11918 // by the time the GC happens, we may not be under BGC anymore. If we try to
11919 // predict often enough it should be ok.
11920 if ((n == max_generation) &&
11921 (recursive_gc_sync::background_running_p()))
11923 n = max_generation - 1;
11924 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11927 if ((n == max_generation) && !local_blocking_collection)
11929 if (!background_allowed_p())
11931 local_blocking_collection = TRUE;
11934 #endif //BACKGROUND_GC
11936 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11938 (local_blocking_collection ? "blocking" : "background")));
11940 if ((n == max_generation) && local_blocking_collection)
11942 alloc_factor = FALSE;
11943 should_notify = TRUE;
11951 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11953 (alloc_factor ? "alloc" : "other"),
11954 dd_collection_count (dynamic_data_of (0)),
11955 new_alloc_remain_percent,
11958 send_full_gc_notification (n_initial, alloc_factor);
11962 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11964 if (!full_gc_approach_event_set)
11966 assert (full_gc_approach_event.IsValid());
11967 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11969 full_gc_end_event.Reset();
11970 full_gc_approach_event.Set();
11971 full_gc_approach_event_set = true;
11975 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11977 if (fgn_maxgen_percent == 0)
11979 return wait_full_gc_na;
11982 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11984 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11986 if (fgn_maxgen_percent == 0)
11988 return wait_full_gc_cancelled;
11991 if (wait_result == WAIT_OBJECT_0)
11993 #ifdef BACKGROUND_GC
11994 if (fgn_last_gc_was_concurrent)
11996 fgn_last_gc_was_concurrent = FALSE;
11997 return wait_full_gc_na;
12000 #endif //BACKGROUND_GC
12002 return wait_full_gc_success;
12007 return wait_full_gc_timeout;
12012 return wait_full_gc_failed;
12016 size_t gc_heap::get_full_compact_gc_count()
12018 return full_gc_counts[gc_type_compacting];
12021 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
12024 BOOL gc_heap::short_on_end_of_seg (int gen_number,
12028 UNREFERENCED_PARAMETER(gen_number);
12029 uint8_t* allocated = heap_segment_allocated(seg);
12031 BOOL sufficient_p = sufficient_space_end_seg (allocated,
12032 heap_segment_reserved (seg),
12033 end_space_after_gc(),
12034 tuning_deciding_short_on_seg);
12037 if (sufficient_gen0_space_p)
12039 dprintf (GTC_LOG, ("gen0 has enough free space"));
12042 sufficient_p = sufficient_gen0_space_p;
12045 return !sufficient_p;
12049 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
12053 BOOL gc_heap::a_fit_free_list_p (int gen_number,
12055 alloc_context* acontext,
12058 BOOL can_fit = FALSE;
12059 generation* gen = generation_of (gen_number);
12060 allocator* gen_allocator = generation_allocator (gen);
12061 size_t sz_list = gen_allocator->first_bucket_size();
12062 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
12064 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
12066 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
12067 uint8_t* prev_free_item = 0;
12069 while (free_list != 0)
12071 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12072 size_t free_list_size = unused_array_size (free_list);
12073 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
12075 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
12076 (size_t)free_list, free_list_size));
12078 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12079 // We ask for more Align (min_obj_size)
12080 // to make sure that we can insert a free object
12081 // in adjust_limit will set the limit lower
12082 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
12084 uint8_t* remain = (free_list + limit);
12085 size_t remain_size = (free_list_size - limit);
12086 if (remain_size >= Align(min_free_list, align_const))
12088 make_unused_array (remain, remain_size);
12089 gen_allocator->thread_item_front (remain, remain_size);
12090 assert (remain_size >= Align (min_obj_size, align_const));
12094 //absorb the entire free list
12095 limit += remain_size;
12097 generation_free_list_space (gen) -= limit;
12099 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12104 else if (gen_allocator->discard_if_no_fit_p())
12106 assert (prev_free_item == 0);
12107 dprintf (3, ("couldn't use this free area, discarding"));
12108 generation_free_obj_space (gen) += free_list_size;
12110 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12111 generation_free_list_space (gen) -= free_list_size;
12115 prev_free_item = free_list;
12117 free_list = free_list_slot (free_list);
12120 sz_list = sz_list * 2;
12127 #ifdef BACKGROUND_GC
12128 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
12130 alloc_context* acontext,
12136 make_unused_array (alloc_start, size);
12138 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
12139 if (g_fEnableAppDomainMonitoring)
12141 GCToEEInterface::RecordAllocatedBytesForHeap(size, heap_number);
12143 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
12145 size_t size_of_array_base = sizeof(ArrayBase);
12147 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
12149 // clear memory while not holding the lock.
12150 size_t size_to_skip = size_of_array_base;
12151 size_t size_to_clear = size - size_to_skip - plug_skew;
12152 size_t saved_size_to_clear = size_to_clear;
12155 uint8_t* end = alloc_start + size - plug_skew;
12156 uint8_t* used = heap_segment_used (seg);
12159 if ((alloc_start + size_to_skip) < used)
12161 size_to_clear = used - (alloc_start + size_to_skip);
12167 dprintf (2, ("bgc loh: setting used to %Ix", end));
12168 heap_segment_used (seg) = end;
12171 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
12172 used, alloc_start, end, size_to_clear));
12176 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
12180 // since we filled in 0xcc for free object when we verify heap,
12181 // we need to make sure we clear those bytes.
12182 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
12184 if (size_to_clear < saved_size_to_clear)
12186 size_to_clear = saved_size_to_clear;
12189 #endif //VERIFY_HEAP
12191 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
12192 add_saved_spinlock_info (true, me_release, mt_clr_large_mem);
12193 leave_spin_lock (&more_space_lock_loh);
12194 memclr (alloc_start + size_to_skip, size_to_clear);
12196 bgc_alloc_lock->loh_alloc_set (alloc_start);
12198 acontext->alloc_ptr = alloc_start;
12199 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
12201 // need to clear the rest of the object before we hand it out.
12202 clear_unused_array(alloc_start, size);
12204 #endif //BACKGROUND_GC
12206 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
12207 alloc_context* acontext,
12210 BOOL can_fit = FALSE;
12211 int gen_number = max_generation + 1;
12212 generation* gen = generation_of (gen_number);
12213 allocator* loh_allocator = generation_allocator (gen);
12215 #ifdef FEATURE_LOH_COMPACTION
12216 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12217 #endif //FEATURE_LOH_COMPACTION
12219 #ifdef BACKGROUND_GC
12221 #endif //BACKGROUND_GC
12222 size_t sz_list = loh_allocator->first_bucket_size();
12223 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
12225 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
12227 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
12228 uint8_t* prev_free_item = 0;
12229 while (free_list != 0)
12231 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12233 size_t free_list_size = unused_array_size(free_list);
12235 #ifdef FEATURE_LOH_COMPACTION
12236 if ((size + loh_pad) <= free_list_size)
12238 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
12239 (size == free_list_size))
12240 #endif //FEATURE_LOH_COMPACTION
12242 #ifdef BACKGROUND_GC
12243 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
12244 bgc_track_loh_alloc();
12245 #endif //BACKGROUND_GC
12247 //unlink the free_item
12248 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12250 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12251 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12252 gen_number, align_const);
12254 #ifdef FEATURE_LOH_COMPACTION
12255 make_unused_array (free_list, loh_pad);
12257 free_list += loh_pad;
12258 free_list_size -= loh_pad;
12259 #endif //FEATURE_LOH_COMPACTION
12261 uint8_t* remain = (free_list + limit);
12262 size_t remain_size = (free_list_size - limit);
12263 if (remain_size != 0)
12265 assert (remain_size >= Align (min_obj_size, align_const));
12266 make_unused_array (remain, remain_size);
12268 if (remain_size >= Align(min_free_list, align_const))
12270 loh_thread_gap_front (remain, remain_size, gen);
12271 assert (remain_size >= Align (min_obj_size, align_const));
12275 generation_free_obj_space (gen) += remain_size;
12277 generation_free_list_space (gen) -= free_list_size;
12278 dprintf (3, ("found fit on loh at %Ix", free_list));
12279 #ifdef BACKGROUND_GC
12282 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12285 #endif //BACKGROUND_GC
12287 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12290 //fix the limit to compensate for adjust_limit_clr making it too short
12291 acontext->alloc_limit += Align (min_obj_size, align_const);
12295 prev_free_item = free_list;
12296 free_list = free_list_slot (free_list);
12299 sz_list = sz_list * 2;
12306 #pragma warning(default:4706)
12309 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12312 alloc_context* acontext,
12314 BOOL* commit_failed_p)
12316 *commit_failed_p = FALSE;
12318 bool hard_limit_short_seg_end_p = false;
12319 #ifdef BACKGROUND_GC
12321 #endif //BACKGROUND_GC
12323 uint8_t*& allocated = ((gen_number == 0) ?
12325 heap_segment_allocated(seg));
12327 size_t pad = Align (min_obj_size, align_const);
12329 #ifdef FEATURE_LOH_COMPACTION
12330 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12331 if (gen_number == (max_generation + 1))
12335 #endif //FEATURE_LOH_COMPACTION
12337 uint8_t* end = heap_segment_committed (seg) - pad;
12339 if (a_size_fit_p (size, allocated, end, align_const))
12341 limit = limit_from_size (size,
12343 gen_number, align_const);
12347 end = heap_segment_reserved (seg) - pad;
12349 if (a_size_fit_p (size, allocated, end, align_const))
12351 limit = limit_from_size (size,
12353 gen_number, align_const);
12355 if (grow_heap_segment (seg, (allocated + limit), &hard_limit_short_seg_end_p))
12361 if (!hard_limit_short_seg_end_p)
12363 dprintf (2, ("can't grow segment, doing a full gc"));
12364 *commit_failed_p = TRUE;
12368 assert (heap_hard_limit);
12377 #ifdef BACKGROUND_GC
12378 if (gen_number != 0)
12380 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12381 bgc_track_loh_alloc();
12383 #endif //BACKGROUND_GC
12385 uint8_t* old_alloc;
12386 old_alloc = allocated;
12387 #ifdef FEATURE_LOH_COMPACTION
12388 if (gen_number == (max_generation + 1))
12390 make_unused_array (old_alloc, loh_pad);
12391 old_alloc += loh_pad;
12392 allocated += loh_pad;
12395 #endif //FEATURE_LOH_COMPACTION
12397 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12398 ((void**) allocated)[-1] = 0; //clear the sync block
12399 #endif //VERIFY_HEAP && _DEBUG
12400 allocated += limit;
12402 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12404 #ifdef BACKGROUND_GC
12407 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12410 #endif //BACKGROUND_GC
12412 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12422 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12424 alloc_context* acontext,
12426 BOOL* commit_failed_p,
12429 *commit_failed_p = FALSE;
12430 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12431 BOOL can_allocate_p = FALSE;
12435 #ifdef BACKGROUND_GC
12436 if (seg->flags & heap_segment_flags_loh_delete)
12438 dprintf (3, ("h%d skipping seg %Ix to be deleted", heap_number, (size_t)seg));
12441 #endif //BACKGROUND_GC
12443 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12444 acontext, align_const, commit_failed_p))
12446 acontext->alloc_limit += Align (min_obj_size, align_const);
12447 can_allocate_p = TRUE;
12451 if (*commit_failed_p)
12453 *oom_r = oom_cant_commit;
12458 seg = heap_segment_next_rw (seg);
12461 return can_allocate_p;
12464 #ifdef BACKGROUND_GC
12466 void gc_heap::wait_for_background (alloc_wait_reason awr, bool loh_p)
12468 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
12470 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12471 add_saved_spinlock_info (loh_p, me_release, mt_wait_bgc);
12472 leave_spin_lock (msl);
12473 background_gc_wait (awr);
12474 enter_spin_lock (msl);
12475 add_saved_spinlock_info (loh_p, me_acquire, mt_wait_bgc);
12478 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr, bool loh_p)
12480 if (recursive_gc_sync::background_running_p())
12482 uint32_t memory_load;
12483 get_memory_info (&memory_load);
12484 if (memory_load >= m_high_memory_load_th)
12486 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12487 wait_for_background (awr, loh_p);
12492 #endif //BACKGROUND_GC
12494 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12495 // return TRUE if that's the case.
12496 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12498 #ifdef BACKGROUND_GC
12499 wait_for_bgc_high_memory (awr_loh_oos_bgc, false);
12500 #endif //BACKGROUND_GC
12502 BOOL did_full_compact_gc = FALSE;
12504 dprintf (2, ("triggering a gen1 GC"));
12505 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12506 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12508 #ifdef MULTIPLE_HEAPS
12509 enter_spin_lock (&more_space_lock_soh);
12510 add_saved_spinlock_info (false, me_acquire, mt_t_eph_gc);
12511 #endif //MULTIPLE_HEAPS
12513 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12515 if (current_full_compact_gc_count > last_full_compact_gc_count)
12517 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12518 did_full_compact_gc = TRUE;
12521 return did_full_compact_gc;
12524 BOOL gc_heap::soh_try_fit (int gen_number,
12526 alloc_context* acontext,
12528 BOOL* commit_failed_p,
12529 BOOL* short_seg_end_p)
12531 BOOL can_allocate = TRUE;
12532 if (short_seg_end_p)
12534 *short_seg_end_p = FALSE;
12537 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12540 if (short_seg_end_p)
12542 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12544 // If the caller doesn't care, we always try to fit at the end of seg;
12545 // otherwise we would only try if we are actually not short at end of seg.
12546 if (!short_seg_end_p || !(*short_seg_end_p))
12548 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12549 acontext, align_const, commit_failed_p);
12553 return can_allocate;
12556 allocation_state gc_heap::allocate_small (int gen_number,
12558 alloc_context* acontext,
12561 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12562 if (recursive_gc_sync::background_running_p())
12564 background_soh_alloc_count++;
12565 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12567 add_saved_spinlock_info (false, me_release, mt_alloc_small);
12568 leave_spin_lock (&more_space_lock_soh);
12569 bool cooperative_mode = enable_preemptive();
12570 GCToOSInterface::Sleep (bgc_alloc_spin);
12571 disable_preemptive (cooperative_mode);
12572 enter_spin_lock (&more_space_lock_soh);
12573 add_saved_spinlock_info (false, me_acquire, mt_alloc_small);
12577 //GCToOSInterface::YieldThread (0);
12580 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12582 gc_reason gr = reason_oos_soh;
12583 oom_reason oom_r = oom_no_failure;
12585 // No variable values should be "carried over" from one state to the other.
12586 // That's why there are local variable for each state
12588 allocation_state soh_alloc_state = a_state_start;
12590 // If we can get a new seg it means allocation will succeed.
12593 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12595 switch (soh_alloc_state)
12597 case a_state_can_allocate:
12598 case a_state_cant_allocate:
12602 case a_state_start:
12604 soh_alloc_state = a_state_try_fit;
12607 case a_state_try_fit:
12609 BOOL commit_failed_p = FALSE;
12610 BOOL can_use_existing_p = FALSE;
12612 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12613 align_const, &commit_failed_p,
12615 soh_alloc_state = (can_use_existing_p ?
12616 a_state_can_allocate :
12618 a_state_trigger_full_compact_gc :
12619 a_state_trigger_ephemeral_gc));
12622 case a_state_try_fit_after_bgc:
12624 BOOL commit_failed_p = FALSE;
12625 BOOL can_use_existing_p = FALSE;
12626 BOOL short_seg_end_p = FALSE;
12628 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12629 align_const, &commit_failed_p,
12631 soh_alloc_state = (can_use_existing_p ?
12632 a_state_can_allocate :
12634 a_state_trigger_2nd_ephemeral_gc :
12635 a_state_trigger_full_compact_gc));
12638 case a_state_try_fit_after_cg:
12640 BOOL commit_failed_p = FALSE;
12641 BOOL can_use_existing_p = FALSE;
12642 BOOL short_seg_end_p = FALSE;
12644 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12645 align_const, &commit_failed_p,
12648 if (can_use_existing_p)
12650 soh_alloc_state = a_state_can_allocate;
12652 #ifdef MULTIPLE_HEAPS
12653 else if (gen0_allocated_after_gc_p)
12655 // some other threads already grabbed the more space lock and allocated
12656 // so we should attempt an ephemeral GC again.
12657 soh_alloc_state = a_state_trigger_ephemeral_gc;
12659 #endif //MULTIPLE_HEAPS
12660 else if (short_seg_end_p)
12662 soh_alloc_state = a_state_cant_allocate;
12663 oom_r = oom_budget;
12667 assert (commit_failed_p);
12668 soh_alloc_state = a_state_cant_allocate;
12669 oom_r = oom_cant_commit;
12673 case a_state_check_and_wait_for_bgc:
12675 BOOL bgc_in_progress_p = FALSE;
12676 BOOL did_full_compacting_gc = FALSE;
12678 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc, false);
12679 soh_alloc_state = (did_full_compacting_gc ?
12680 a_state_try_fit_after_cg :
12681 a_state_try_fit_after_bgc);
12684 case a_state_trigger_ephemeral_gc:
12686 BOOL commit_failed_p = FALSE;
12687 BOOL can_use_existing_p = FALSE;
12688 BOOL short_seg_end_p = FALSE;
12689 BOOL bgc_in_progress_p = FALSE;
12690 BOOL did_full_compacting_gc = FALSE;
12692 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12693 if (did_full_compacting_gc)
12695 soh_alloc_state = a_state_try_fit_after_cg;
12699 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12700 align_const, &commit_failed_p,
12702 #ifdef BACKGROUND_GC
12703 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12704 #endif //BACKGROUND_GC
12706 if (can_use_existing_p)
12708 soh_alloc_state = a_state_can_allocate;
12712 if (short_seg_end_p)
12714 if (should_expand_in_full_gc)
12716 dprintf (2, ("gen1 GC wanted to expand!"));
12717 soh_alloc_state = a_state_trigger_full_compact_gc;
12721 soh_alloc_state = (bgc_in_progress_p ?
12722 a_state_check_and_wait_for_bgc :
12723 a_state_trigger_full_compact_gc);
12726 else if (commit_failed_p)
12728 soh_alloc_state = a_state_trigger_full_compact_gc;
12732 #ifdef MULTIPLE_HEAPS
12733 // some other threads already grabbed the more space lock and allocated
12734 // so we should attempt an ephemeral GC again.
12735 assert (gen0_allocated_after_gc_p);
12736 soh_alloc_state = a_state_trigger_ephemeral_gc;
12737 #else //MULTIPLE_HEAPS
12738 assert (!"shouldn't get here");
12739 #endif //MULTIPLE_HEAPS
12745 case a_state_trigger_2nd_ephemeral_gc:
12747 BOOL commit_failed_p = FALSE;
12748 BOOL can_use_existing_p = FALSE;
12749 BOOL short_seg_end_p = FALSE;
12750 BOOL did_full_compacting_gc = FALSE;
12753 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12755 if (did_full_compacting_gc)
12757 soh_alloc_state = a_state_try_fit_after_cg;
12761 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12762 align_const, &commit_failed_p,
12764 if (short_seg_end_p || commit_failed_p)
12766 soh_alloc_state = a_state_trigger_full_compact_gc;
12770 assert (can_use_existing_p);
12771 soh_alloc_state = a_state_can_allocate;
12776 case a_state_trigger_full_compact_gc:
12778 if (fgn_maxgen_percent)
12780 dprintf (2, ("FGN: SOH doing last GC before we throw OOM"));
12781 send_full_gc_notification (max_generation, FALSE);
12784 BOOL got_full_compacting_gc = FALSE;
12786 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, false);
12787 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12792 assert (!"Invalid state!");
12799 if (soh_alloc_state == a_state_cant_allocate)
12801 assert (oom_r != oom_no_failure);
12802 handle_oom (heap_number,
12805 heap_segment_allocated (ephemeral_heap_segment),
12806 heap_segment_reserved (ephemeral_heap_segment));
12808 add_saved_spinlock_info (false, me_release, mt_alloc_small_cant);
12809 leave_spin_lock (&more_space_lock_soh);
12812 assert ((soh_alloc_state == a_state_can_allocate) ||
12813 (soh_alloc_state == a_state_cant_allocate) ||
12814 (soh_alloc_state == a_state_retry_allocate));
12816 return soh_alloc_state;
12819 #ifdef BACKGROUND_GC
12821 void gc_heap::bgc_track_loh_alloc()
12823 if (current_c_gc_state == c_gc_state_planning)
12825 Interlocked::Increment (&loh_alloc_thread_count);
12826 dprintf (3, ("h%d: inc lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12831 void gc_heap::bgc_untrack_loh_alloc()
12833 if (current_c_gc_state == c_gc_state_planning)
12835 Interlocked::Decrement (&loh_alloc_thread_count);
12836 dprintf (3, ("h%d: dec lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12840 BOOL gc_heap::bgc_loh_should_allocate()
12842 size_t min_gc_size = dd_min_size (dynamic_data_of (max_generation + 1));
12844 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12849 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12851 if ((bgc_begin_loh_size / end_loh_size) > 2)
12853 dprintf (3, ("alloc-ed too much before bgc started"));
12857 dprintf (3, ("alloc-ed too much after bgc started"));
12863 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12867 #endif //BACKGROUND_GC
12869 size_t gc_heap::get_large_seg_size (size_t size)
12871 size_t default_seg_size = min_loh_segment_size;
12872 #ifdef SEG_MAPPING_TABLE
12873 size_t align_size = default_seg_size;
12874 #else //SEG_MAPPING_TABLE
12875 size_t align_size = default_seg_size / 2;
12876 #endif //SEG_MAPPING_TABLE
12877 int align_const = get_alignment_constant (FALSE);
12878 size_t large_seg_size = align_on_page (
12879 max (default_seg_size,
12880 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12881 align_size) / align_size * align_size)));
12882 return large_seg_size;
12885 BOOL gc_heap::loh_get_new_seg (generation* gen,
12888 BOOL* did_full_compact_gc,
12891 UNREFERENCED_PARAMETER(gen);
12892 UNREFERENCED_PARAMETER(align_const);
12894 *did_full_compact_gc = FALSE;
12896 size_t seg_size = get_large_seg_size (size);
12898 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12902 loh_alloc_since_cg += seg_size;
12909 return (new_seg != 0);
12912 // PERF TODO: this is too aggressive; and in hard limit we should
12913 // count the actual allocated bytes instead of only updating it during
12914 // getting a new seg.
12915 BOOL gc_heap::retry_full_compact_gc (size_t size)
12917 size_t seg_size = get_large_seg_size (size);
12919 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12924 #ifdef MULTIPLE_HEAPS
12925 uint64_t total_alloc_size = 0;
12926 for (int i = 0; i < n_heaps; i++)
12928 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12931 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12935 #endif //MULTIPLE_HEAPS
12940 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12941 BOOL* did_full_compact_gc,
12944 BOOL bgc_in_progress = FALSE;
12945 *did_full_compact_gc = FALSE;
12946 #ifdef BACKGROUND_GC
12947 if (recursive_gc_sync::background_running_p())
12949 bgc_in_progress = TRUE;
12950 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12951 wait_for_background (awr, loh_p);
12952 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12953 if (current_full_compact_gc_count > last_full_compact_gc_count)
12955 *did_full_compact_gc = TRUE;
12958 #endif //BACKGROUND_GC
12960 return bgc_in_progress;
12963 BOOL gc_heap::loh_try_fit (int gen_number,
12965 alloc_context* acontext,
12967 BOOL* commit_failed_p,
12970 BOOL can_allocate = TRUE;
12972 if (!a_fit_free_list_large_p (size, acontext, align_const))
12974 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12975 acontext, align_const,
12976 commit_failed_p, oom_r);
12978 #ifdef BACKGROUND_GC
12979 if (can_allocate && recursive_gc_sync::background_running_p())
12981 bgc_loh_size_increased += size;
12983 #endif //BACKGROUND_GC
12985 #ifdef BACKGROUND_GC
12988 if (recursive_gc_sync::background_running_p())
12990 bgc_loh_allocated_in_free += size;
12993 #endif //BACKGROUND_GC
12995 return can_allocate;
12998 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
13002 BOOL did_full_compact_gc = FALSE;
13004 size_t last_full_compact_gc_count = get_full_compact_gc_count();
13006 // Set this so the next GC will be a full compacting GC.
13007 if (!last_gc_before_oom)
13009 last_gc_before_oom = TRUE;
13012 #ifdef BACKGROUND_GC
13013 if (recursive_gc_sync::background_running_p())
13015 wait_for_background (((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc), loh_p);
13016 dprintf (2, ("waited for BGC - done"));
13018 #endif //BACKGROUND_GC
13020 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13021 size_t current_full_compact_gc_count = get_full_compact_gc_count();
13022 if (current_full_compact_gc_count > last_full_compact_gc_count)
13024 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
13025 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13026 did_full_compact_gc = TRUE;
13030 dprintf (3, ("h%d full GC", heap_number));
13032 trigger_gc_for_alloc (max_generation, gr, msl, loh_p, mt_t_full_gc);
13034 current_full_compact_gc_count = get_full_compact_gc_count();
13036 if (current_full_compact_gc_count == last_full_compact_gc_count)
13038 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
13039 // We requested a full GC but didn't get because of the elevation logic
13040 // which means we should fail.
13041 *oom_r = oom_unproductive_full_gc;
13045 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
13047 last_full_compact_gc_count,
13048 current_full_compact_gc_count));
13050 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13051 did_full_compact_gc = TRUE;
13055 return did_full_compact_gc;
13058 #ifdef RECORD_LOH_STATE
13059 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
13061 // When the state is can_allocate we already have released the more
13062 // space lock. So we are not logging states here since this code
13063 // is not thread safe.
13064 if (loh_state_to_save != a_state_can_allocate)
13066 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
13067 last_loh_states[loh_state_index].thread_id = thread_id;
13070 if (loh_state_index == max_saved_loh_states)
13072 loh_state_index = 0;
13075 assert (loh_state_index < max_saved_loh_states);
13078 #endif //RECORD_LOH_STATE
13080 bool gc_heap::should_retry_other_heap (size_t size)
13082 #ifdef MULTIPLE_HEAPS
13083 if (heap_hard_limit)
13085 size_t total_heap_committed_recorded =
13086 current_total_committed - current_total_committed_bookkeeping;
13087 size_t min_size = dd_min_size (g_heaps[0]->dynamic_data_of (max_generation + 1));
13088 size_t slack_space = max (commit_min_th, min_size);
13089 bool retry_p = ((total_heap_committed_recorded + size) < (heap_hard_limit - slack_space));
13090 dprintf (1, ("%Id - %Id - total committed %Id - size %Id = %Id, %s",
13091 heap_hard_limit, slack_space, total_heap_committed_recorded, size,
13092 (heap_hard_limit - slack_space - total_heap_committed_recorded - size),
13093 (retry_p ? "retry" : "no retry")));
13097 #endif //MULTIPLE_HEAPS
13103 allocation_state gc_heap::allocate_large (int gen_number,
13105 alloc_context* acontext,
13108 #ifdef BACKGROUND_GC
13109 if (recursive_gc_sync::background_running_p())
13111 background_loh_alloc_count++;
13112 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
13114 if (bgc_loh_should_allocate())
13116 if (!bgc_alloc_spin_loh)
13118 add_saved_spinlock_info (true, me_release, mt_alloc_large);
13119 leave_spin_lock (&more_space_lock_loh);
13120 bool cooperative_mode = enable_preemptive();
13121 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
13122 disable_preemptive (cooperative_mode);
13123 enter_spin_lock (&more_space_lock_loh);
13124 add_saved_spinlock_info (true, me_acquire, mt_alloc_large);
13125 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
13130 wait_for_background (awr_loh_alloc_during_bgc, true);
13134 #endif //BACKGROUND_GC
13136 gc_reason gr = reason_oos_loh;
13137 generation* gen = generation_of (gen_number);
13138 oom_reason oom_r = oom_no_failure;
13139 size_t current_full_compact_gc_count = 0;
13141 // No variable values should be "carried over" from one state to the other.
13142 // That's why there are local variable for each state
13143 allocation_state loh_alloc_state = a_state_start;
13144 #ifdef RECORD_LOH_STATE
13145 EEThreadId current_thread_id;
13146 current_thread_id.SetToCurrentThread();
13147 #endif //RECORD_LOH_STATE
13149 // If we can get a new seg it means allocation will succeed.
13152 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
13154 #ifdef RECORD_LOH_STATE
13155 add_saved_loh_state (loh_alloc_state, current_thread_id);
13156 #endif //RECORD_LOH_STATE
13157 switch (loh_alloc_state)
13159 case a_state_can_allocate:
13160 case a_state_cant_allocate:
13164 case a_state_start:
13166 loh_alloc_state = a_state_try_fit;
13169 case a_state_try_fit:
13171 BOOL commit_failed_p = FALSE;
13172 BOOL can_use_existing_p = FALSE;
13174 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13175 align_const, &commit_failed_p, &oom_r);
13176 loh_alloc_state = (can_use_existing_p ?
13177 a_state_can_allocate :
13179 a_state_trigger_full_compact_gc :
13180 a_state_acquire_seg));
13181 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13184 case a_state_try_fit_new_seg:
13186 BOOL commit_failed_p = FALSE;
13187 BOOL can_use_existing_p = FALSE;
13189 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13190 align_const, &commit_failed_p, &oom_r);
13191 // Even after we got a new seg it doesn't necessarily mean we can allocate,
13192 // another LOH allocating thread could have beat us to acquire the msl so
13193 // we need to try again.
13194 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
13195 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13198 case a_state_try_fit_after_cg:
13200 BOOL commit_failed_p = FALSE;
13201 BOOL can_use_existing_p = FALSE;
13203 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13204 align_const, &commit_failed_p, &oom_r);
13205 // If we failed to commit, we bail right away 'cause we already did a
13206 // full compacting GC.
13207 loh_alloc_state = (can_use_existing_p ?
13208 a_state_can_allocate :
13210 a_state_cant_allocate :
13211 a_state_acquire_seg_after_cg));
13212 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13215 case a_state_try_fit_after_bgc:
13217 BOOL commit_failed_p = FALSE;
13218 BOOL can_use_existing_p = FALSE;
13220 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13221 align_const, &commit_failed_p, &oom_r);
13222 loh_alloc_state = (can_use_existing_p ?
13223 a_state_can_allocate :
13225 a_state_trigger_full_compact_gc :
13226 a_state_acquire_seg_after_bgc));
13227 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13230 case a_state_acquire_seg:
13232 BOOL can_get_new_seg_p = FALSE;
13233 BOOL did_full_compacting_gc = FALSE;
13235 current_full_compact_gc_count = get_full_compact_gc_count();
13237 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13238 loh_alloc_state = (can_get_new_seg_p ?
13239 a_state_try_fit_new_seg :
13240 (did_full_compacting_gc ?
13241 a_state_check_retry_seg :
13242 a_state_check_and_wait_for_bgc));
13245 case a_state_acquire_seg_after_cg:
13247 BOOL can_get_new_seg_p = FALSE;
13248 BOOL did_full_compacting_gc = FALSE;
13250 current_full_compact_gc_count = get_full_compact_gc_count();
13252 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13253 // Since we release the msl before we try to allocate a seg, other
13254 // threads could have allocated a bunch of segments before us so
13255 // we might need to retry.
13256 loh_alloc_state = (can_get_new_seg_p ?
13257 a_state_try_fit_after_cg :
13258 a_state_check_retry_seg);
13261 case a_state_acquire_seg_after_bgc:
13263 BOOL can_get_new_seg_p = FALSE;
13264 BOOL did_full_compacting_gc = FALSE;
13266 current_full_compact_gc_count = get_full_compact_gc_count();
13268 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13269 loh_alloc_state = (can_get_new_seg_p ?
13270 a_state_try_fit_new_seg :
13271 (did_full_compacting_gc ?
13272 a_state_check_retry_seg :
13273 a_state_trigger_full_compact_gc));
13274 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13277 case a_state_check_and_wait_for_bgc:
13279 BOOL bgc_in_progress_p = FALSE;
13280 BOOL did_full_compacting_gc = FALSE;
13282 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc, true);
13283 loh_alloc_state = (!bgc_in_progress_p ?
13284 a_state_trigger_full_compact_gc :
13285 (did_full_compacting_gc ?
13286 a_state_try_fit_after_cg :
13287 a_state_try_fit_after_bgc));
13290 case a_state_trigger_full_compact_gc:
13292 if (fgn_maxgen_percent)
13294 dprintf (2, ("FGN: LOH doing last GC before we throw OOM"));
13295 send_full_gc_notification (max_generation, FALSE);
13298 BOOL got_full_compacting_gc = FALSE;
13300 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, true);
13301 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13302 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13305 case a_state_check_retry_seg:
13307 BOOL should_retry_gc = retry_full_compact_gc (size);
13308 BOOL should_retry_get_seg = FALSE;
13309 if (!should_retry_gc)
13311 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13312 current_full_compact_gc_count = get_full_compact_gc_count();
13313 if (current_full_compact_gc_count > last_full_compact_gc_count)
13315 should_retry_get_seg = TRUE;
13319 loh_alloc_state = (should_retry_gc ?
13320 a_state_trigger_full_compact_gc :
13321 (should_retry_get_seg ?
13322 a_state_try_fit_after_cg :
13323 a_state_cant_allocate));
13324 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13329 assert (!"Invalid state!");
13336 if (loh_alloc_state == a_state_cant_allocate)
13338 assert (oom_r != oom_no_failure);
13339 if (should_retry_other_heap (size))
13341 loh_alloc_state = a_state_retry_allocate;
13345 handle_oom (heap_number,
13351 add_saved_spinlock_info (true, me_release, mt_alloc_large_cant);
13352 leave_spin_lock (&more_space_lock_loh);
13355 assert ((loh_alloc_state == a_state_can_allocate) ||
13356 (loh_alloc_state == a_state_cant_allocate) ||
13357 (loh_alloc_state == a_state_retry_allocate));
13358 return loh_alloc_state;
13361 // BGC's final mark phase will acquire the msl, so release it here and re-acquire.
13362 void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr,
13363 GCSpinLock* msl, bool loh_p,
13364 msl_take_state take_state)
13366 #ifdef BACKGROUND_GC
13369 add_saved_spinlock_info (loh_p, me_release, take_state);
13370 leave_spin_lock (msl);
13372 #endif //BACKGROUND_GC
13374 vm_heap->GarbageCollectGeneration (gen_number, gr);
13376 #ifdef MULTIPLE_HEAPS
13379 enter_spin_lock (msl);
13380 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13382 #endif //MULTIPLE_HEAPS
13384 #ifdef BACKGROUND_GC
13387 enter_spin_lock (msl);
13388 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13390 #endif //BACKGROUND_GC
13393 allocation_state gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13396 if (gc_heap::gc_started)
13398 wait_for_gc_done();
13399 return a_state_retry_allocate;
13402 bool loh_p = (gen_number > 0);
13403 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13405 #ifdef SYNCHRONIZATION_STATS
13406 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13407 #endif //SYNCHRONIZATION_STATS
13408 enter_spin_lock (msl);
13409 add_saved_spinlock_info (loh_p, me_acquire, mt_try_alloc);
13410 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13411 #ifdef SYNCHRONIZATION_STATS
13412 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13413 total_msl_acquire += msl_acquire;
13414 num_msl_acquired++;
13415 if (msl_acquire > 200)
13417 num_high_msl_acquire++;
13421 num_low_msl_acquire++;
13423 #endif //SYNCHRONIZATION_STATS
13426 // We are commenting this out 'cause we don't see the point - we already
13427 // have checked gc_started when we were acquiring the msl - no need to check
13428 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13429 // need to release msl which causes all sorts of trouble.
13430 if (gc_heap::gc_started)
13432 #ifdef SYNCHRONIZATION_STATS
13434 #endif //SYNCHRONIZATION_STATS
13435 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13438 //Rendez vous early (MP scaling issue)
13439 //dprintf (1, ("[%d]waiting for gc", heap_number));
13440 wait_for_gc_done();
13441 #ifdef MULTIPLE_HEAPS
13443 #endif //MULTIPLE_HEAPS
13448 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13450 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13452 if (fgn_maxgen_percent)
13454 check_for_full_gc (gen_number, size);
13457 if (!(new_allocation_allowed (gen_number)))
13459 if (fgn_maxgen_percent && (gen_number == 0))
13461 // We only check gen0 every so often, so take this opportunity to check again.
13462 check_for_full_gc (gen_number, size);
13465 #ifdef BACKGROUND_GC
13466 wait_for_bgc_high_memory (awr_gen0_alloc, loh_p);
13467 #endif //BACKGROUND_GC
13469 #ifdef SYNCHRONIZATION_STATS
13471 #endif //SYNCHRONIZATION_STATS
13472 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13474 if (!settings.concurrent || (gen_number == 0))
13476 trigger_gc_for_alloc (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh),
13477 msl, loh_p, mt_try_budget);
13481 allocation_state can_allocate = ((gen_number == 0) ?
13482 allocate_small (gen_number, size, acontext, align_const) :
13483 allocate_large (gen_number, size, acontext, align_const));
13485 if (can_allocate == a_state_can_allocate)
13487 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13488 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13490 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13492 allocated_since_last_gc += alloc_context_bytes;
13494 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13496 #ifdef FEATURE_REDHAWK
13497 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13498 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13500 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13501 // The ones that do are much less efficient.
13502 #if defined(FEATURE_EVENT_TRACE)
13503 if (EVENT_ENABLED(GCAllocationTick_V3))
13505 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13507 #endif //FEATURE_EVENT_TRACE
13508 #endif //FEATURE_REDHAWK
13509 etw_allocation_running_amount[etw_allocation_index] = 0;
13513 return can_allocate;
13516 #ifdef MULTIPLE_HEAPS
13517 void gc_heap::balance_heaps (alloc_context* acontext)
13519 if (acontext->alloc_count < 4)
13521 if (acontext->alloc_count == 0)
13523 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13524 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13525 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13526 acontext->set_alloc_heap(acontext->get_home_heap());
13527 hp->alloc_context_count++;
13532 BOOL set_home_heap = FALSE;
13535 if (heap_select::can_find_heap_fast())
13537 if (acontext->get_home_heap() != NULL)
13538 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13539 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13541 set_home_heap = TRUE;
13547 if ((acontext->alloc_count & 3) == 0)
13548 set_home_heap = TRUE;
13554 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13555 if (n_heaps > MAX_SUPPORTED_CPUS)
13557 // on machines with many processors cache affinity is really king, so don't even try
13558 // to balance on these.
13559 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13560 acontext->alloc_heap = acontext->home_heap;
13565 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13567 dynamic_data* dd = org_hp->dynamic_data_of (0);
13568 ptrdiff_t org_size = dd_new_allocation (dd);
13569 int org_alloc_context_count;
13570 int max_alloc_context_count;
13572 ptrdiff_t max_size;
13573 size_t delta = dd_min_size (dd)/4;
13575 int start, end, finish;
13576 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13577 finish = start + n_heaps;
13583 max_size = org_size + delta;
13584 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13586 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13587 max_size = max_size + delta;
13589 org_alloc_context_count = org_hp->alloc_context_count;
13590 max_alloc_context_count = org_alloc_context_count;
13591 if (max_alloc_context_count > 1)
13592 max_size /= max_alloc_context_count;
13594 for (int i = start; i < end; i++)
13596 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13597 dd = hp->dynamic_data_of (0);
13598 ptrdiff_t size = dd_new_allocation (dd);
13599 if (hp == acontext->get_home_heap()->pGenGCHeap)
13600 size = size + delta;
13601 int hp_alloc_context_count = hp->alloc_context_count;
13602 if (hp_alloc_context_count > 0)
13603 size /= (hp_alloc_context_count + 1);
13604 if (size > max_size)
13608 max_alloc_context_count = hp_alloc_context_count;
13612 while (org_alloc_context_count != org_hp->alloc_context_count ||
13613 max_alloc_context_count != max_hp->alloc_context_count);
13615 if ((max_hp == org_hp) && (end < finish))
13617 start = end; end = finish;
13618 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13622 if (max_hp != org_hp)
13624 org_hp->alloc_context_count--;
13625 max_hp->alloc_context_count++;
13626 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13627 if (!gc_thread_no_affinitize_p)
13629 if (GCToOSInterface::CanEnableGCCPUGroups())
13630 { //only set ideal processor when max_hp and org_hp are in the same cpu
13631 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13632 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13633 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13634 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13636 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13638 GCThreadAffinity affinity;
13639 affinity.Processor = group_proc_no;
13640 affinity.Group = org_gn;
13641 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13643 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13644 org_hp->heap_number));
13650 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13652 GCThreadAffinity affinity;
13653 affinity.Processor = proc_no;
13654 affinity.Group = GCThreadAffinity::None;
13656 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13658 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13659 org_hp->heap_number));
13663 dprintf (3, ("Switching context %p (home heap %d) ",
13665 acontext->get_home_heap()->pGenGCHeap->heap_number));
13666 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13667 org_hp->heap_number,
13669 org_alloc_context_count));
13670 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13671 max_hp->heap_number,
13672 dd_new_allocation(max_hp->dynamic_data_of(0)),
13673 max_alloc_context_count));
13678 acontext->alloc_count++;
13681 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t alloc_size)
13683 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13684 dprintf (3, ("[h%d] LA: %Id", org_hp->heap_number, alloc_size));
13686 //if (size > 128*1024)
13689 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13691 ptrdiff_t org_size = dd_new_allocation (dd);
13693 ptrdiff_t max_size;
13694 size_t delta = dd_min_size (dd) * 4;
13696 int start, end, finish;
13697 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13698 finish = start + n_heaps;
13703 max_size = org_size + delta;
13704 dprintf (3, ("orig hp: %d, max size: %d",
13705 org_hp->heap_number,
13708 for (int i = start; i < end; i++)
13710 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13711 dd = hp->dynamic_data_of (max_generation + 1);
13712 ptrdiff_t size = dd_new_allocation (dd);
13713 dprintf (3, ("hp: %d, size: %d",
13716 if (size > max_size)
13720 dprintf (3, ("max hp: %d, max size: %d",
13721 max_hp->heap_number,
13727 if ((max_hp == org_hp) && (end < finish))
13729 start = end; end = finish;
13730 delta = dd_min_size(dd) * 4; // Need to tuning delta
13734 if (max_hp != org_hp)
13736 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13737 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13738 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13748 #endif //MULTIPLE_HEAPS
13750 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13751 int alloc_generation_number)
13753 allocation_state status;
13756 #ifdef MULTIPLE_HEAPS
13757 if (alloc_generation_number == 0)
13759 balance_heaps (acontext);
13760 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13764 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13765 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13766 if (status == a_state_retry_allocate)
13768 dprintf (3, ("LOH h%d alloc retry!", alloc_heap->heap_number));
13772 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13773 #endif //MULTIPLE_HEAPS
13775 while (status == a_state_retry_allocate);
13777 return (status == a_state_can_allocate);
13781 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13783 size_t size = Align (jsize);
13784 assert (size >= Align (min_obj_size));
13787 uint8_t* result = acontext->alloc_ptr;
13788 acontext->alloc_ptr+=size;
13789 if (acontext->alloc_ptr <= acontext->alloc_limit)
13791 CObjectHeader* obj = (CObjectHeader*)result;
13797 acontext->alloc_ptr -= size;
13800 #pragma inline_depth(0)
13803 if (! allocate_more_space (acontext, size, 0))
13807 #pragma inline_depth(20)
13815 void gc_heap::leave_allocation_segment (generation* gen)
13817 adjust_limit (0, 0, gen, max_generation);
13820 void gc_heap::init_free_and_plug()
13822 #ifdef FREE_USAGE_STATS
13823 for (int i = 0; i <= settings.condemned_generation; i++)
13825 generation* gen = generation_of (i);
13826 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13827 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13828 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13831 if (settings.condemned_generation != max_generation)
13833 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13835 generation* gen = generation_of (i);
13836 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13839 #endif //FREE_USAGE_STATS
13842 void gc_heap::print_free_and_plug (const char* msg)
13844 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13845 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13846 for (int i = 0; i <= older_gen; i++)
13848 generation* gen = generation_of (i);
13849 for (int j = 0; j < NUM_GEN_POWER2; j++)
13851 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13853 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13856 (settings.concurrent ? "BGC" : "GC"),
13859 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13864 UNREFERENCED_PARAMETER(msg);
13865 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13868 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13870 #ifdef FREE_USAGE_STATS
13871 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13872 generation* gen = generation_of (gen_number);
13873 size_t sz = BASE_GEN_SIZE;
13876 for (; i < NUM_GEN_POWER2; i++)
13878 if (plug_size < sz)
13885 (gen->gen_plugs[i])++;
13887 UNREFERENCED_PARAMETER(gen_number);
13888 UNREFERENCED_PARAMETER(plug_size);
13889 #endif //FREE_USAGE_STATS
13892 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13894 #ifdef FREE_USAGE_STATS
13895 generation* gen = generation_of (gen_number);
13896 size_t sz = BASE_GEN_SIZE;
13899 for (; i < NUM_GEN_POWER2; i++)
13901 if (free_size < sz)
13908 (gen->gen_current_pinned_free_spaces[i])++;
13909 generation_pinned_free_obj_space (gen) += free_size;
13910 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13911 free_size, (i + 10), gen_number,
13912 generation_pinned_free_obj_space (gen),
13913 gen->gen_current_pinned_free_spaces[i]));
13915 UNREFERENCED_PARAMETER(gen_number);
13916 UNREFERENCED_PARAMETER(free_size);
13917 #endif //FREE_USAGE_STATS
13920 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13922 #ifdef FREE_USAGE_STATS
13923 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13924 generation* gen = generation_of (gen_number);
13925 size_t sz = BASE_GEN_SIZE;
13928 for (; i < NUM_GEN_POWER2; i++)
13930 if (free_size < sz)
13937 (gen->gen_free_spaces[i])++;
13939 UNREFERENCED_PARAMETER(gen_number);
13940 UNREFERENCED_PARAMETER(free_size);
13941 #endif //FREE_USAGE_STATS
13944 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13946 #ifdef FREE_USAGE_STATS
13947 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13948 generation* gen = generation_of (gen_number);
13949 size_t sz = BASE_GEN_SIZE;
13952 for (; i < NUM_GEN_POWER2; i++)
13954 if (free_size < sz)
13961 (gen->gen_free_spaces[i])--;
13963 UNREFERENCED_PARAMETER(gen_number);
13964 UNREFERENCED_PARAMETER(free_size);
13965 #endif //FREE_USAGE_STATS
13968 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13969 int from_gen_number,
13970 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13972 size = Align (size);
13973 assert (size >= Align (min_obj_size));
13974 assert (from_gen_number < max_generation);
13975 assert (from_gen_number >= 0);
13976 assert (generation_of (from_gen_number + 1) == gen);
13978 allocator* gen_allocator = generation_allocator (gen);
13979 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13980 int pad_in_front = ((old_loc != 0) && ((from_gen_number+1) != max_generation)) ? USE_PADDING_FRONT : 0;
13982 size_t real_size = size + Align (min_obj_size);
13984 real_size += Align (min_obj_size);
13986 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13987 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13989 size_t sz_list = gen_allocator->first_bucket_size();
13990 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13992 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13994 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13995 uint8_t* prev_free_item = 0;
13996 while (free_list != 0)
13998 dprintf (3, ("considering free list %Ix", (size_t)free_list));
14000 size_t free_list_size = unused_array_size (free_list);
14002 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
14003 old_loc, USE_PADDING_TAIL | pad_in_front))
14005 dprintf (4, ("F:%Ix-%Id",
14006 (size_t)free_list, free_list_size));
14008 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
14009 generation_free_list_space (gen) -= free_list_size;
14010 remove_gen_free (gen->gen_num, free_list_size);
14012 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
14013 generation_allocate_end_seg_p (gen) = FALSE;
14016 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
14017 else if (discard_p || (a_l_idx == 0))
14019 dprintf (3, ("couldn't use this free area, discarding"));
14020 generation_free_obj_space (gen) += free_list_size;
14022 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
14023 generation_free_list_space (gen) -= free_list_size;
14024 remove_gen_free (gen->gen_num, free_list_size);
14028 prev_free_item = free_list;
14030 free_list = free_list_slot (free_list);
14033 sz_list = sz_list * 2;
14035 //go back to the beginning of the segment list
14036 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
14037 if (seg != generation_allocation_segment (gen))
14039 leave_allocation_segment (gen);
14040 generation_allocation_segment (gen) = seg;
14042 while (seg != ephemeral_heap_segment)
14044 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14045 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
14047 dprintf (3, ("using what's left in committed"));
14048 adjust_limit (heap_segment_plan_allocated (seg),
14049 heap_segment_committed (seg) -
14050 heap_segment_plan_allocated (seg),
14051 gen, from_gen_number+1);
14052 generation_allocate_end_seg_p (gen) = TRUE;
14053 // dformat (t, 3, "Expanding segment allocation");
14054 heap_segment_plan_allocated (seg) =
14055 heap_segment_committed (seg);
14060 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14061 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14062 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
14064 dprintf (3, ("using what's left in reserved"));
14065 adjust_limit (heap_segment_plan_allocated (seg),
14066 heap_segment_committed (seg) -
14067 heap_segment_plan_allocated (seg),
14068 gen, from_gen_number+1);
14069 generation_allocate_end_seg_p (gen) = TRUE;
14070 heap_segment_plan_allocated (seg) =
14071 heap_segment_committed (seg);
14077 leave_allocation_segment (gen);
14078 heap_segment* next_seg = heap_segment_next_rw (seg);
14081 dprintf (3, ("getting next segment"));
14082 generation_allocation_segment (gen) = next_seg;
14083 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14084 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14093 seg = generation_allocation_segment (gen);
14095 //No need to fix the last region. Will be done later
14106 uint8_t* result = generation_allocation_pointer (gen);
14110 if ((pad_in_front & USE_PADDING_FRONT) &&
14111 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14112 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14114 pad = Align (min_obj_size);
14115 set_plug_padded (old_loc);
14117 #endif //SHORT_PLUGS
14119 #ifdef FEATURE_STRUCTALIGN
14120 _ASSERTE(!old_loc || alignmentOffset != 0);
14121 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14124 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14125 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14128 #else // FEATURE_STRUCTALIGN
14129 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14131 pad += switch_alignment_size (is_plug_padded (old_loc));
14132 set_node_realigned (old_loc);
14133 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14134 (size_t)old_loc, (size_t)(result+pad)));
14135 assert (same_large_alignment_p (result + pad, old_loc));
14137 #endif // FEATURE_STRUCTALIGN
14138 dprintf (3, ("Allocate %Id bytes", size));
14140 if ((old_loc == 0) || (pad != 0))
14142 //allocating a non plug or a gap, so reset the start region
14143 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14146 generation_allocation_pointer (gen) += size + pad;
14147 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14148 if (generation_allocate_end_seg_p (gen))
14150 generation_end_seg_allocated (gen) += size;
14154 generation_free_list_allocated (gen) += size;
14156 generation_allocation_size (gen) += size;
14158 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
14159 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14160 generation_allocation_context_start_region (gen)));
14162 return result + pad;;
14166 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
14168 //make sure that every generation has a planned allocation start
14169 int gen_number = max_generation - 1;
14170 while (gen_number>= 0)
14172 generation* gen = generation_of (gen_number);
14173 if (0 == generation_plan_allocation_start (gen))
14175 realloc_plan_generation_start (gen, consing_gen);
14177 assert (generation_plan_allocation_start (gen));
14182 // now we know the planned allocation size
14183 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
14184 heap_segment* seg = generation_allocation_segment (consing_gen);
14185 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
14189 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14194 assert (settings.condemned_generation == max_generation);
14195 uint8_t* first_address = generation_allocation_limit (consing_gen);
14196 //look through the pinned plugs for relevant ones.
14197 //Look for the right pinned plug to start from.
14200 while (mi != mark_stack_tos)
14202 m = pinned_plug_of (mi);
14203 if ((pinned_plug (m) == first_address))
14208 assert (mi != mark_stack_tos);
14209 pinned_len (m) = size;
14213 //tododefrag optimize for new segment (plan_allocated == mem)
14214 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
14219 BOOL set_padding_on_saved_p,
14220 mark* pinned_plug_entry,
14221 #endif //SHORT_PLUGS
14222 BOOL consider_bestfit,
14223 int active_new_gen_number
14224 REQD_ALIGN_AND_OFFSET_DCL)
14226 UNREFERENCED_PARAMETER(active_new_gen_number);
14227 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
14229 size = Align (size);
14230 assert (size >= Align (min_obj_size));
14231 int pad_in_front = ((old_loc != 0) && (active_new_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14233 if (consider_bestfit && use_bestfit)
14235 assert (bestfit_seg);
14236 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
14238 return bestfit_seg->fit (old_loc,
14240 set_padding_on_saved_p,
14242 #endif //SHORT_PLUGS
14243 size REQD_ALIGN_AND_OFFSET_ARG);
14246 heap_segment* seg = generation_allocation_segment (gen);
14248 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14249 generation_allocation_limit (gen), old_loc,
14250 ((generation_allocation_limit (gen) !=
14251 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
14253 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
14254 generation_allocation_limit (gen)));
14257 uint8_t* first_address = (generation_allocation_limit (gen) ?
14258 generation_allocation_limit (gen) :
14259 heap_segment_mem (seg));
14260 assert (in_range_for_segment (first_address, seg));
14262 uint8_t* end_address = heap_segment_reserved (seg);
14264 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
14265 first_address, generation_allocation_limit (gen), end_address));
14270 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
14272 assert (settings.condemned_generation == max_generation);
14273 //look through the pinned plugs for relevant ones.
14274 //Look for the right pinned plug to start from.
14275 while (mi != mark_stack_tos)
14277 m = pinned_plug_of (mi);
14278 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14280 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14286 if (mi != mark_stack_tos)
14288 //fix old free list.
14289 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14291 dprintf(3,("gc filling up hole"));
14292 ptrdiff_t mi1 = (ptrdiff_t)mi;
14293 while ((mi1 >= 0) &&
14294 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14296 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14301 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14302 pinned_len (pinned_plug_of(mi1)) = hsize;
14303 dprintf (3, ("changing %Ix len %Ix->%Ix",
14304 pinned_plug (pinned_plug_of(mi1)),
14305 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14312 assert (generation_allocation_limit (gen) ==
14313 generation_allocation_pointer (gen));
14314 mi = mark_stack_tos;
14317 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14319 size_t len = pinned_len (m);
14320 uint8_t* free_list = (pinned_plug (m) - len);
14321 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14322 free_list, (free_list + len), len));
14323 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14325 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14326 (size_t)free_list, len));
14328 generation_allocation_pointer (gen) = free_list;
14329 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14330 generation_allocation_limit (gen) = (free_list + len);
14332 goto allocate_in_free;
14335 m = pinned_plug_of (mi);
14338 //switch to the end of the segment.
14339 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14340 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14341 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14342 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14343 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14344 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14345 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14347 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14348 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14350 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14351 generation_allocation_limit (gen)));
14352 assert (!"Can't allocate if no free space");
14363 uint8_t* result = generation_allocation_pointer (gen);
14367 if ((pad_in_front & USE_PADDING_FRONT) &&
14368 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14369 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14372 pad = Align (min_obj_size);
14373 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14375 #endif //SHORT_PLUGS
14377 #ifdef FEATURE_STRUCTALIGN
14378 _ASSERTE(!old_loc || alignmentOffset != 0);
14379 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14382 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14383 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14387 #else // FEATURE_STRUCTALIGN
14388 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14390 pad += switch_alignment_size (is_plug_padded (old_loc));
14391 set_node_realigned (old_loc);
14392 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14393 (size_t)old_loc, (size_t)(result+pad)));
14394 assert (same_large_alignment_p (result + pad, old_loc));
14397 #endif // FEATURE_STRUCTALIGN
14399 if ((old_loc == 0) || (pad != 0))
14401 //allocating a non plug or a gap, so reset the start region
14402 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14405 generation_allocation_pointer (gen) += size + pad;
14406 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14407 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14409 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14410 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14411 generation_allocation_context_start_region (gen)));
14413 return result + pad;
14417 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14419 heap_segment* seg = generation_allocation_segment (consing_gen);
14420 if (seg != ephemeral_heap_segment)
14422 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14423 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14425 //fix the allocated size of the segment.
14426 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14428 generation* new_consing_gen = generation_of (max_generation - 1);
14429 generation_allocation_pointer (new_consing_gen) =
14430 heap_segment_mem (ephemeral_heap_segment);
14431 generation_allocation_limit (new_consing_gen) =
14432 generation_allocation_pointer (new_consing_gen);
14433 generation_allocation_context_start_region (new_consing_gen) =
14434 generation_allocation_pointer (new_consing_gen);
14435 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14437 return new_consing_gen;
14440 return consing_gen;
14443 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14445 int from_gen_number,
14447 BOOL* convert_to_pinned_p,
14448 uint8_t* next_pinned_plug,
14449 heap_segment* current_seg,
14450 #endif //SHORT_PLUGS
14452 REQD_ALIGN_AND_OFFSET_DCL)
14454 // Make sure that the youngest generation gap hasn't been allocated
14455 if (settings.promotion)
14457 assert (generation_plan_allocation_start (youngest_generation) == 0);
14460 size = Align (size);
14461 assert (size >= Align (min_obj_size));
14462 int to_gen_number = from_gen_number;
14463 if (from_gen_number != (int)max_generation)
14465 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14468 dprintf (3, ("aic gen%d: s: %Id", gen->gen_num, size));
14470 int pad_in_front = ((old_loc != 0) && (to_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14472 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14474 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14475 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14479 heap_segment* seg = generation_allocation_segment (gen);
14480 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14481 generation_allocation_limit (gen), old_loc,
14482 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14484 if ((! (pinned_plug_que_empty_p()) &&
14485 (generation_allocation_limit (gen) ==
14486 pinned_plug (oldest_pin()))))
14488 size_t entry = deque_pinned_plug();
14489 mark* pinned_plug_entry = pinned_plug_of (entry);
14490 size_t len = pinned_len (pinned_plug_entry);
14491 uint8_t* plug = pinned_plug (pinned_plug_entry);
14492 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14494 #ifdef FREE_USAGE_STATS
14495 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14496 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14497 generation_allocated_since_last_pin (gen),
14499 generation_allocated_in_pinned_free (gen)));
14500 generation_allocated_since_last_pin (gen) = 0;
14502 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14503 #endif //FREE_USAGE_STATS
14505 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14506 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14508 assert(mark_stack_array[entry].len == 0 ||
14509 mark_stack_array[entry].len >= Align(min_obj_size));
14510 generation_allocation_pointer (gen) = plug + len;
14511 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14512 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14513 set_allocator_next_pin (gen);
14515 //Add the size of the pinned plug to the right pinned allocations
14516 //find out which gen this pinned plug came from
14517 int frgn = object_gennum (plug);
14518 if ((frgn != (int)max_generation) && settings.promotion)
14520 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14521 int togn = object_gennum_plan (plug);
14524 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14530 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14532 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14533 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14537 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14539 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14540 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14541 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14545 #ifndef RESPECT_LARGE_ALIGNMENT
14546 assert (gen != youngest_generation);
14547 #endif //RESPECT_LARGE_ALIGNMENT
14549 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14550 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14551 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14552 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14554 dprintf (3, ("Expanded segment allocation by committing more memory"));
14555 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14556 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14560 heap_segment* next_seg = heap_segment_next (seg);
14561 assert (generation_allocation_pointer (gen)>=
14562 heap_segment_mem (seg));
14563 // Verify that all pinned plugs for this segment are consumed
14564 if (!pinned_plug_que_empty_p() &&
14565 ((pinned_plug (oldest_pin()) <
14566 heap_segment_allocated (seg)) &&
14567 (pinned_plug (oldest_pin()) >=
14568 generation_allocation_pointer (gen))))
14570 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14571 pinned_plug (oldest_pin())));
14574 assert (generation_allocation_pointer (gen)>=
14575 heap_segment_mem (seg));
14576 assert (generation_allocation_pointer (gen)<=
14577 heap_segment_committed (seg));
14578 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14582 generation_allocation_segment (gen) = next_seg;
14583 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14584 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14585 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14589 return 0; //should only happen during allocation of generation 0 gap
14590 // in that case we are going to grow the heap anyway
14595 set_allocator_next_pin (gen);
14602 assert (generation_allocation_pointer (gen)>=
14603 heap_segment_mem (generation_allocation_segment (gen)));
14604 uint8_t* result = generation_allocation_pointer (gen);
14607 if ((pad_in_front & USE_PADDING_FRONT) &&
14608 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14609 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14611 ptrdiff_t dist = old_loc - result;
14614 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14619 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14621 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14625 pad = Align (min_obj_size);
14626 set_plug_padded (old_loc);
14629 #endif //SHORT_PLUGS
14630 #ifdef FEATURE_STRUCTALIGN
14631 _ASSERTE(!old_loc || alignmentOffset != 0);
14632 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14633 if ((old_loc != 0))
14635 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14636 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14639 #else // FEATURE_STRUCTALIGN
14640 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14642 pad += switch_alignment_size (is_plug_padded (old_loc));
14643 set_node_realigned(old_loc);
14644 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14645 (size_t)old_loc, (size_t)(result+pad)));
14646 assert (same_large_alignment_p (result + pad, old_loc));
14648 #endif // FEATURE_STRUCTALIGN
14651 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14653 assert (old_loc != 0);
14654 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14655 assert (dist_to_next_pin >= 0);
14657 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14659 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14661 generation_allocation_pointer (gen),
14662 generation_allocation_limit (gen),
14665 dist_to_next_pin));
14666 clear_plug_padded (old_loc);
14668 *convert_to_pinned_p = TRUE;
14669 record_interesting_data_point (idp_converted_pin);
14674 #endif //SHORT_PLUGS
14676 if ((old_loc == 0) || (pad != 0))
14678 //allocating a non plug or a gap, so reset the start region
14679 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14682 generation_allocation_pointer (gen) += size + pad;
14683 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14685 #ifdef FREE_USAGE_STATS
14686 generation_allocated_since_last_pin (gen) += size;
14687 #endif //FREE_USAGE_STATS
14689 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14690 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14691 generation_allocation_context_start_region (gen)));
14693 assert (result + pad);
14694 return result + pad;
14698 inline int power (int x, int y)
14701 for (int i = 0; i < y; i++)
14708 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14711 BOOL* blocking_collection_p
14712 STRESS_HEAP_ARG(int n_original))
14714 int n = current_gen;
14715 #ifdef MULTIPLE_HEAPS
14716 BOOL joined_last_gc_before_oom = FALSE;
14717 for (int i = 0; i < n_heaps; i++)
14719 if (g_heaps[i]->last_gc_before_oom)
14721 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14722 joined_last_gc_before_oom = TRUE;
14727 BOOL joined_last_gc_before_oom = last_gc_before_oom;
14728 #endif //MULTIPLE_HEAPS
14730 if (joined_last_gc_before_oom && settings.pause_mode != pause_low_latency)
14732 assert (*blocking_collection_p);
14735 if (should_evaluate_elevation && (n == max_generation))
14737 dprintf (GTC_LOG, ("lock: %d(%d)",
14738 (settings.should_lock_elevation ? 1 : 0),
14739 settings.elevation_locked_count));
14741 if (settings.should_lock_elevation)
14743 settings.elevation_locked_count++;
14744 if (settings.elevation_locked_count == 6)
14746 settings.elevation_locked_count = 0;
14750 n = max_generation - 1;
14751 settings.elevation_reduced = TRUE;
14756 settings.elevation_locked_count = 0;
14761 settings.should_lock_elevation = FALSE;
14762 settings.elevation_locked_count = 0;
14765 if (provisional_mode_triggered && (n == max_generation))
14767 // There are a few cases where we should not reduce the generation.
14768 if ((initial_gen == max_generation) || (settings.reason == reason_alloc_loh))
14770 // If we are doing a full GC in the provisional mode, we always
14771 // make it blocking because we don't want to get into a situation
14772 // where foreground GCs are asking for a compacting full GC right away
14773 // and not getting it.
14774 dprintf (GTC_LOG, ("full GC induced, not reducing gen"));
14775 *blocking_collection_p = TRUE;
14777 else if (should_expand_in_full_gc || joined_last_gc_before_oom)
14779 dprintf (GTC_LOG, ("need full blocking GCs to expand heap or avoid OOM, not reducing gen"));
14780 assert (*blocking_collection_p);
14784 dprintf (GTC_LOG, ("reducing gen in PM: %d->%d->%d", initial_gen, n, (max_generation - 1)));
14785 n = max_generation - 1;
14789 if (should_expand_in_full_gc)
14791 should_expand_in_full_gc = FALSE;
14794 if (heap_hard_limit)
14796 // If we have already consumed 90% of the limit, we should check to see if we should compact LOH.
14797 // TODO: should unify this with gen2.
14798 dprintf (GTC_LOG, ("committed %Id is %d%% of limit %Id",
14799 current_total_committed, (int)((float)current_total_committed * 100.0 / (float)heap_hard_limit),
14801 if ((current_total_committed * 10) >= (heap_hard_limit * 9))
14803 bool full_compact_gc_p = false;
14805 size_t loh_frag = get_total_gen_fragmentation (max_generation + 1);
14807 // If the LOH frag is >= 1/8 it's worth compacting it
14808 if ((loh_frag * 8) >= heap_hard_limit)
14810 dprintf (GTC_LOG, ("loh frag: %Id > 1/8 of limit %Id", loh_frag, (heap_hard_limit / 8)));
14811 full_compact_gc_p = true;
14815 // If there's not much fragmentation but it looks like it'll be productive to
14816 // collect LOH, do that.
14817 size_t est_loh_reclaim = get_total_gen_estimated_reclaim (max_generation + 1);
14818 full_compact_gc_p = ((est_loh_reclaim * 8) >= heap_hard_limit);
14819 dprintf (GTC_LOG, ("loh est reclaim: %Id, 1/8 of limit %Id", est_loh_reclaim, (heap_hard_limit / 8)));
14822 if (full_compact_gc_p)
14824 n = max_generation;
14825 *blocking_collection_p = TRUE;
14826 settings.loh_compaction = TRUE;
14827 dprintf (GTC_LOG, ("compacting LOH due to hard limit"));
14832 if ((n == max_generation) && (*blocking_collection_p == FALSE))
14834 // If we are doing a gen2 we should reset elevation regardless and let the gen2
14835 // decide if we should lock again or in the bgc case by design we will not retract
14837 settings.should_lock_elevation = FALSE;
14838 settings.elevation_locked_count = 0;
14839 dprintf (1, ("doing bgc, reset elevation"));
14843 #ifdef BACKGROUND_GC
14844 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14845 // generations to be collected,
14847 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14848 // things that need to be fixed in this code block.
14849 if (n_original != max_generation &&
14850 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14852 #ifndef FEATURE_REDHAWK
14853 // for the GC stress mix mode throttle down gen2 collections
14854 if (g_pConfig->IsGCStressMix())
14856 size_t current_gc_count = 0;
14858 #ifdef MULTIPLE_HEAPS
14859 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14861 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14862 #endif //MULTIPLE_HEAPS
14863 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14864 if ((current_gc_count % 10) == 0)
14866 n = max_generation;
14869 // for traditional GC stress
14871 #endif // !FEATURE_REDHAWK
14872 if (*blocking_collection_p)
14874 // We call StressHeap() a lot for Concurrent GC Stress. However,
14875 // if we can not do a concurrent collection, no need to stress anymore.
14876 // @TODO: Enable stress when the memory pressure goes down again
14877 GCStressPolicy::GlobalDisable();
14881 n = max_generation;
14884 #endif //BACKGROUND_GC
14885 #endif //STRESS_HEAP
14891 size_t get_survived_size (gc_history_per_heap* hist)
14893 size_t surv_size = 0;
14894 gc_generation_data* gen_data;
14896 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14898 gen_data = &(hist->gen_data[gen_number]);
14899 surv_size += (gen_data->size_after -
14900 gen_data->free_list_space_after -
14901 gen_data->free_obj_space_after);
14907 size_t gc_heap::get_total_survived_size()
14909 size_t total_surv_size = 0;
14910 #ifdef MULTIPLE_HEAPS
14911 for (int i = 0; i < gc_heap::n_heaps; i++)
14913 gc_heap* hp = gc_heap::g_heaps[i];
14914 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14915 total_surv_size += get_survived_size (current_gc_data_per_heap);
14918 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14919 total_surv_size = get_survived_size (current_gc_data_per_heap);
14920 #endif //MULTIPLE_HEAPS
14921 return total_surv_size;
14924 size_t gc_heap::get_total_allocated_since_last_gc()
14926 size_t total_allocated_size = 0;
14927 #ifdef MULTIPLE_HEAPS
14928 for (int i = 0; i < gc_heap::n_heaps; i++)
14930 gc_heap* hp = gc_heap::g_heaps[i];
14931 total_allocated_size += hp->allocated_since_last_gc;
14932 hp->allocated_since_last_gc = 0;
14935 total_allocated_size = allocated_since_last_gc;
14936 allocated_since_last_gc = 0;
14937 #endif //MULTIPLE_HEAPS
14938 return total_allocated_size;
14941 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14942 size_t gc_heap::get_current_allocated()
14944 dynamic_data* dd = dynamic_data_of (0);
14945 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14946 dd = dynamic_data_of (max_generation + 1);
14947 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14949 return current_alloc;
14952 size_t gc_heap::get_total_allocated()
14954 size_t total_current_allocated = 0;
14955 #ifdef MULTIPLE_HEAPS
14956 for (int i = 0; i < gc_heap::n_heaps; i++)
14958 gc_heap* hp = gc_heap::g_heaps[i];
14959 total_current_allocated += hp->get_current_allocated();
14962 total_current_allocated = get_current_allocated();
14963 #endif //MULTIPLE_HEAPS
14964 return total_current_allocated;
14967 size_t gc_heap::current_generation_size (int gen_number)
14969 dynamic_data* dd = dynamic_data_of (gen_number);
14970 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14971 - dd_new_allocation (dd));
14977 #pragma warning(push)
14978 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14982 This is called by when we are actually doing a GC, or when we are just checking whether
14983 we would do a full blocking GC, in which case check_only_p is TRUE.
14985 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14987 settings.reason is ignored
14988 budgets are not checked (since they are checked before this is called)
14989 it doesn't change anything non local like generation_skip_ratio
14991 int gc_heap::generation_to_condemn (int n_initial,
14992 BOOL* blocking_collection_p,
14993 BOOL* elevation_requested_p,
14996 gc_mechanisms temp_settings = settings;
14997 gen_to_condemn_tuning temp_condemn_reasons;
14998 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14999 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
15002 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
15004 assert (n_initial >= 1);
15007 assert (settings.reason != reason_empty);
15010 local_condemn_reasons->init();
15014 if (heap_number == 0)
15016 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
15020 BOOL low_memory_detected = g_low_memory_status;
15021 uint32_t memory_load = 0;
15022 uint64_t available_physical = 0;
15023 uint64_t available_page_file = 0;
15024 BOOL check_memory = FALSE;
15025 BOOL high_fragmentation = FALSE;
15026 BOOL v_high_memory_load = FALSE;
15027 BOOL high_memory_load = FALSE;
15028 BOOL low_ephemeral_space = FALSE;
15029 BOOL evaluate_elevation = TRUE;
15030 *elevation_requested_p = FALSE;
15031 *blocking_collection_p = FALSE;
15033 BOOL check_max_gen_alloc = TRUE;
15037 #endif //STRESS_HEAP
15041 dd_fragmentation (dynamic_data_of (0)) =
15042 generation_free_list_space (youngest_generation) +
15043 generation_free_obj_space (youngest_generation);
15045 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
15046 generation_free_list_space (large_object_generation) +
15047 generation_free_obj_space (large_object_generation);
15049 //save new_allocation
15050 for (i = 0; i <= max_generation+1; i++)
15052 dynamic_data* dd = dynamic_data_of (i);
15053 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
15055 dd_new_allocation (dd),
15056 dd_desired_allocation (dd)));
15057 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
15060 local_condemn_reasons->set_gen (gen_initial, n);
15063 #ifdef BACKGROUND_GC
15064 if (recursive_gc_sync::background_running_p())
15066 dprintf (GTC_LOG, ("bgc in prog, 1"));
15067 check_max_gen_alloc = FALSE;
15069 #endif //BACKGROUND_GC
15071 if (check_max_gen_alloc)
15073 //figure out if large objects need to be collected.
15074 if (get_new_allocation (max_generation+1) <= 0)
15076 n = max_generation;
15077 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15081 //figure out which generation ran out of allocation
15082 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
15084 if (get_new_allocation (i) <= 0)
15095 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15098 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
15102 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
15103 //time based tuning
15104 // if enough time has elapsed since the last gc
15105 // and the number of gc is too low (1/10 of lower gen) then collect
15106 // This should also be enabled if we have memory concerns
15107 int n_time_max = max_generation;
15111 if (recursive_gc_sync::background_running_p())
15113 n_time_max = max_generation - 1;
15117 if ((local_settings->pause_mode == pause_interactive) ||
15118 (local_settings->pause_mode == pause_sustained_low_latency))
15120 dynamic_data* dd0 = dynamic_data_of (0);
15121 size_t now = GetHighPrecisionTimeStamp();
15123 for (i = (temp_gen+1); i <= n_time_max; i++)
15125 dynamic_data* dd = dynamic_data_of (i);
15126 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
15127 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
15128 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
15130 n = min (i, n_time_max);
15131 dprintf (GTC_LOG, ("time %d", n));
15136 local_condemn_reasons->set_gen (gen_time_tuning, n);
15142 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
15144 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
15146 if (n < (max_generation - 1))
15148 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
15150 n = max (n, max_generation - 1);
15151 local_settings->promotion = TRUE;
15152 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
15153 heap_number, generation_skip_ratio, n));
15154 local_condemn_reasons->set_condition (gen_low_card_p);
15160 generation_skip_ratio = 100;
15163 if (dt_low_ephemeral_space_p (check_only_p ?
15164 tuning_deciding_full_gc :
15165 tuning_deciding_condemned_gen))
15167 low_ephemeral_space = TRUE;
15169 n = max (n, max_generation - 1);
15170 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
15171 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
15173 if (!provisional_mode_triggered)
15175 #ifdef BACKGROUND_GC
15176 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
15177 #endif //BACKGROUND_GC
15179 //It is better to defragment first if we are running out of space for
15180 //the ephemeral generation but we have enough fragmentation to make up for it
15181 //in the non ephemeral generation. Essentially we are trading a gen2 for
15182 // having to expand heap in ephemeral collections.
15183 if (dt_high_frag_p (tuning_deciding_condemned_gen,
15184 max_generation - 1,
15187 high_fragmentation = TRUE;
15188 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
15189 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
15195 //figure out which ephemeral generation is too fragramented
15197 for (i = n+1; i < max_generation; i++)
15199 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
15201 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
15208 if (low_ephemeral_space)
15211 local_settings->promotion = TRUE;
15216 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
15221 if (settings.pause_mode == pause_low_latency)
15223 if (!is_induced (settings.reason))
15225 n = min (n, max_generation - 1);
15226 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
15227 evaluate_elevation = FALSE;
15233 // It's hard to catch when we get to the point that the memory load is so high
15234 // we get an induced GC from the finalizer thread so we are checking the memory load
15235 // for every gen0 GC.
15236 check_memory = (check_only_p ?
15238 ((n >= 1) || low_memory_detected));
15242 //find out if we are short on memory
15243 get_memory_info (&memory_load, &available_physical, &available_page_file);
15244 if (heap_number == 0)
15246 dprintf (GTC_LOG, ("ml: %d", memory_load));
15249 // Need to get it early enough for all heaps to use.
15250 entry_available_physical_mem = available_physical;
15251 local_settings->entry_memory_load = memory_load;
15253 // @TODO: Force compaction more often under GCSTRESS
15254 if (memory_load >= high_memory_load_th || low_memory_detected)
15256 #ifdef SIMPLE_DPRINTF
15257 // stress log can't handle any parameter that's bigger than a void*.
15258 if (heap_number == 0)
15260 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
15262 #endif //SIMPLE_DPRINTF
15264 high_memory_load = TRUE;
15266 if (memory_load >= v_high_memory_load_th || low_memory_detected)
15268 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
15269 // gen1/gen0 may take a lot more memory than gen2.
15270 if (!high_fragmentation)
15272 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
15274 v_high_memory_load = TRUE;
15278 if (!high_fragmentation)
15280 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
15284 if (high_fragmentation)
15286 if (high_memory_load)
15288 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
15290 else if (v_high_memory_load)
15292 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
15298 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
15299 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
15300 high_fragmentation));
15302 if (should_expand_in_full_gc)
15304 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
15305 *blocking_collection_p = TRUE;
15306 evaluate_elevation = FALSE;
15307 n = max_generation;
15308 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
15311 if (last_gc_before_oom)
15313 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
15314 n = max_generation;
15315 *blocking_collection_p = TRUE;
15317 if ((local_settings->reason == reason_oos_loh) ||
15318 (local_settings->reason == reason_alloc_loh))
15320 evaluate_elevation = FALSE;
15323 local_condemn_reasons->set_condition (gen_before_oom);
15328 if (is_induced_blocking (settings.reason) &&
15329 n_initial == max_generation
15330 IN_STRESS_HEAP( && !settings.stress_induced ))
15332 if (heap_number == 0)
15334 dprintf (GTC_LOG, ("induced - BLOCK"));
15337 *blocking_collection_p = TRUE;
15338 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
15339 evaluate_elevation = FALSE;
15342 if (settings.reason == reason_induced_noforce)
15344 local_condemn_reasons->set_condition (gen_induced_noforce_p);
15345 evaluate_elevation = FALSE;
15349 if (!provisional_mode_triggered && evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
15351 *elevation_requested_p = TRUE;
15353 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
15354 if (high_memory_load || v_high_memory_load)
15356 dynamic_data* dd_max = dynamic_data_of (max_generation);
15357 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
15359 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
15360 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
15361 n = max_generation;
15362 local_condemn_reasons->set_condition (gen_almost_max_alloc);
15366 if (n <= max_generation)
15369 if (high_fragmentation)
15371 //elevate to max_generation
15372 n = max_generation;
15373 dprintf (GTC_LOG, ("h%d: f full", heap_number));
15375 #ifdef BACKGROUND_GC
15376 if (high_memory_load || v_high_memory_load)
15378 // For background GC we want to do blocking collections more eagerly because we don't
15379 // want to get into the situation where the memory load becomes high while we are in
15380 // a background GC and we'd have to wait for the background GC to finish to start
15381 // a blocking collection (right now the implemenation doesn't handle converting
15382 // a background GC to a blocking collection midway.
15383 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15384 *blocking_collection_p = TRUE;
15387 if (v_high_memory_load)
15389 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15390 *blocking_collection_p = TRUE;
15392 #endif //BACKGROUND_GC
15396 n = max (n, max_generation - 1);
15397 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15404 if (!provisional_mode_triggered && (n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15406 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15407 heap_number, n_alloc));
15408 if (get_new_allocation (max_generation) <= 0)
15410 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15411 n = max_generation;
15412 local_condemn_reasons->set_condition (gen_max_gen1);
15416 //figure out if max_generation is too fragmented -> blocking collection
15417 if (!provisional_mode_triggered && (n == max_generation))
15419 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15421 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15422 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15423 if (local_settings->pause_mode != pause_sustained_low_latency)
15425 *blocking_collection_p = TRUE;
15430 #ifdef BACKGROUND_GC
15431 if (n == max_generation)
15433 if (heap_number == 0)
15435 BOOL bgc_heap_too_small = TRUE;
15436 size_t gen2size = 0;
15437 size_t gen3size = 0;
15438 #ifdef MULTIPLE_HEAPS
15439 for (int i = 0; i < n_heaps; i++)
15441 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15442 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15444 bgc_heap_too_small = FALSE;
15448 #else //MULTIPLE_HEAPS
15449 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15450 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15452 bgc_heap_too_small = FALSE;
15454 #endif //MULTIPLE_HEAPS
15456 if (bgc_heap_too_small)
15458 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15461 // do not turn stress-induced collections into blocking GCs
15462 if (!settings.stress_induced)
15463 #endif //STRESS_HEAP
15465 *blocking_collection_p = TRUE;
15468 local_condemn_reasons->set_condition (gen_gen2_too_small);
15472 #endif //BACKGROUND_GC
15478 #ifdef BACKGROUND_GC
15479 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15480 // generations to be collected,
15482 if (orig_gen != max_generation &&
15483 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15485 *elevation_requested_p = FALSE;
15487 #endif //BACKGROUND_GC
15488 #endif //STRESS_HEAP
15492 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15495 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15496 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15499 local_condemn_reasons->print (heap_number);
15502 if ((local_settings->reason == reason_oos_soh) ||
15503 (local_settings->reason == reason_oos_loh))
15513 #pragma warning(pop)
15517 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15519 // if the memory load is higher, the threshold we'd want to collect gets lower.
15520 size_t min_mem_based_on_available =
15521 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15523 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15524 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15526 #ifdef SIMPLE_DPRINTF
15527 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15528 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15529 #endif //SIMPLE_DPRINTF
15530 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15534 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15536 return min (available_mem, (256*1024*1024)) / num_heaps;
15540 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15544 #ifdef BACKGROUND_GC
15545 void gc_heap::init_background_gc ()
15547 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15548 generation* gen = generation_of (max_generation);
15549 generation_allocation_pointer (gen)= 0;
15550 generation_allocation_limit (gen) = 0;
15551 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15553 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15555 //reset the plan allocation for each segment
15556 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15557 seg = heap_segment_next_rw (seg))
15559 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15562 if (heap_number == 0)
15564 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15566 background_saved_lowest_address,
15567 background_saved_highest_address));
15570 gc_lh_block_event.Reset();
15573 #endif //BACKGROUND_GC
15576 void fire_drain_mark_list_event (size_t mark_list_objects)
15578 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15582 void fire_revisit_event (size_t dirtied_pages,
15583 size_t marked_objects,
15584 BOOL large_objects_p)
15586 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15590 void fire_overflow_event (uint8_t* overflow_min,
15591 uint8_t* overflow_max,
15592 size_t marked_objects,
15593 int large_objects_p)
15595 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15598 void gc_heap::concurrent_print_time_delta (const char* msg)
15601 size_t current_time = GetHighPrecisionTimeStamp();
15602 size_t elapsed_time = current_time - time_bgc_last;
15603 time_bgc_last = current_time;
15605 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15607 UNREFERENCED_PARAMETER(msg);
15611 void gc_heap::free_list_info (int gen_num, const char* msg)
15613 UNREFERENCED_PARAMETER(gen_num);
15614 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15615 dprintf (3, ("h%d: %s", heap_number, msg));
15616 for (int i = 0; i <= (max_generation + 1); i++)
15618 generation* gen = generation_of (i);
15619 if ((generation_allocation_size (gen) == 0) &&
15620 (generation_free_list_space (gen) == 0) &&
15621 (generation_free_obj_space (gen) == 0))
15623 // don't print if everything is 0.
15627 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15629 generation_allocation_size (gen),
15630 generation_free_list_space (gen),
15631 generation_free_obj_space (gen)));
15635 UNREFERENCED_PARAMETER(msg);
15636 #endif // BACKGROUND_GC && TRACE_GC
15639 void gc_heap::update_collection_counts_for_no_gc()
15641 assert (settings.pause_mode == pause_no_gc);
15643 settings.condemned_generation = max_generation;
15644 #ifdef MULTIPLE_HEAPS
15645 for (int i = 0; i < n_heaps; i++)
15646 g_heaps[i]->update_collection_counts();
15647 #else //MULTIPLE_HEAPS
15648 update_collection_counts();
15649 #endif //MULTIPLE_HEAPS
15651 full_gc_counts[gc_type_blocking]++;
15654 BOOL gc_heap::should_proceed_with_gc()
15656 if (gc_heap::settings.pause_mode == pause_no_gc)
15658 if (current_no_gc_region_info.started)
15660 // The no_gc mode was already in progress yet we triggered another GC,
15661 // this effectively exits the no_gc mode.
15662 restore_data_for_no_gc();
15665 return should_proceed_for_no_gc();
15671 //internal part of gc used by the serial and concurrent version
15672 void gc_heap::gc1()
15674 #ifdef BACKGROUND_GC
15675 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15676 #endif //BACKGROUND_GC
15679 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15682 verify_soh_segment_list();
15684 int n = settings.condemned_generation;
15686 if (settings.reason == reason_pm_full_gc)
15688 assert (n == max_generation);
15691 gen_to_condemn_tuning* local_condemn_reasons = &(get_gc_data_per_heap()->gen_to_condemn_reasons);
15692 local_condemn_reasons->init();
15693 local_condemn_reasons->set_gen (gen_initial, n);
15694 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15697 update_collection_counts ();
15699 #ifdef BACKGROUND_GC
15700 bgc_alloc_lock->check();
15701 #endif //BACKGROUND_GC
15703 free_list_info (max_generation, "beginning");
15705 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15707 assert (g_gc_card_table == card_table);
15709 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15710 assert (g_gc_card_bundle_table == card_bundle_table);
15714 if (n == max_generation)
15716 gc_low = lowest_address;
15717 gc_high = highest_address;
15721 gc_low = generation_allocation_start (generation_of (n));
15722 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15724 #ifdef BACKGROUND_GC
15725 if (settings.concurrent)
15728 time_bgc_last = GetHighPrecisionTimeStamp();
15731 FIRE_EVENT(BGCBegin);
15733 concurrent_print_time_delta ("BGC");
15735 //#ifdef WRITE_WATCH
15736 //reset_write_watch (FALSE);
15737 //#endif //WRITE_WATCH
15739 concurrent_print_time_delta ("RW");
15740 background_mark_phase();
15741 free_list_info (max_generation, "after mark phase");
15743 background_sweep();
15744 free_list_info (max_generation, "after sweep phase");
15747 #endif //BACKGROUND_GC
15749 mark_phase (n, FALSE);
15751 GCScan::GcRuntimeStructuresValid (FALSE);
15753 GCScan::GcRuntimeStructuresValid (TRUE);
15757 size_t end_gc_time = GetHighPrecisionTimeStamp();
15758 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15760 //adjust the allocation size from the pinned quantities.
15761 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15763 generation* gn = generation_of (gen_number);
15764 if (settings.compaction)
15766 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15767 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15771 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15772 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15774 generation_pinned_allocation_sweep_size (gn) = 0;
15775 generation_pinned_allocation_compact_size (gn) = 0;
15778 #ifdef BACKGROUND_GC
15779 if (settings.concurrent)
15781 dynamic_data* dd = dynamic_data_of (n);
15782 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15784 free_list_info (max_generation, "after computing new dynamic data");
15786 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15788 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15790 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15791 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15792 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15793 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15794 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15798 #endif //BACKGROUND_GC
15800 free_list_info (max_generation, "end");
15801 for (int gen_number = 0; gen_number <= n; gen_number++)
15803 dynamic_data* dd = dynamic_data_of (gen_number);
15804 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15805 compute_new_dynamic_data (gen_number);
15808 if (n != max_generation)
15810 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15811 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15813 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15814 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15815 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15819 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15821 free_list_info (max_generation, "after computing new dynamic data");
15823 if (heap_number == 0)
15825 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15826 dd_collection_count (dynamic_data_of (0)),
15827 settings.condemned_generation,
15828 dd_gc_elapsed_time (dynamic_data_of (0))));
15831 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15833 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15834 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15838 if (n < max_generation)
15840 compute_promoted_allocation (1 + n);
15842 dynamic_data* dd = dynamic_data_of (1 + n);
15843 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15844 generation_free_obj_space (generation_of (1 + n));
15846 #ifdef BACKGROUND_GC
15847 if (current_c_gc_state != c_gc_state_planning)
15848 #endif //BACKGROUND_GC
15850 if (settings.promotion)
15852 dd_fragmentation (dd) = new_fragmentation;
15856 //assert (dd_fragmentation (dd) == new_fragmentation);
15861 #ifdef BACKGROUND_GC
15862 if (!settings.concurrent)
15863 #endif //BACKGROUND_GC
15865 #ifndef FEATURE_REDHAWK
15866 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15867 assert(GCToEEInterface::IsGCThread());
15868 #endif // FEATURE_REDHAWK
15869 adjust_ephemeral_limits();
15872 #ifdef BACKGROUND_GC
15873 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15874 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15875 #endif //BACKGROUND_GC
15877 if (fgn_maxgen_percent)
15879 if (settings.condemned_generation == (max_generation - 1))
15881 check_for_full_gc (max_generation - 1, 0);
15883 else if (settings.condemned_generation == max_generation)
15885 if (full_gc_approach_event_set
15886 #ifdef MULTIPLE_HEAPS
15887 && (heap_number == 0)
15888 #endif //MULTIPLE_HEAPS
15891 dprintf (2, ("FGN-GC: setting gen2 end event"));
15893 full_gc_approach_event.Reset();
15894 #ifdef BACKGROUND_GC
15895 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15896 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15897 #endif //BACKGROUND_GC
15898 full_gc_end_event.Set();
15899 full_gc_approach_event_set = false;
15904 #ifdef BACKGROUND_GC
15905 if (!settings.concurrent)
15906 #endif //BACKGROUND_GC
15908 //decide on the next allocation quantum
15909 if (alloc_contexts_used >= 1)
15911 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15912 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15913 get_alignment_constant(FALSE));
15914 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15918 descr_generations (FALSE);
15920 verify_soh_segment_list();
15922 #ifdef BACKGROUND_GC
15923 add_to_history_per_heap();
15924 if (heap_number == 0)
15928 #endif // BACKGROUND_GC
15931 if (GCStatistics::Enabled() && heap_number == 0)
15932 g_GCStatistics.AddGCStats(settings,
15933 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15937 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15938 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15941 #ifdef BACKGROUND_GC
15942 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15943 #endif //BACKGROUND_GC
15945 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15948 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15949 // value. If we ever allow randomly adjusting this as the process runs,
15950 // we cannot call it this way as joins need to match - we must have the same
15951 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15952 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15954 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15955 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15959 #ifdef BACKGROUND_GC
15960 bool cooperative_mode = true;
15962 if (settings.concurrent)
15964 cooperative_mode = enable_preemptive ();
15966 #ifdef MULTIPLE_HEAPS
15967 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15968 if (bgc_t_join.joined())
15970 bgc_threads_sync_event.Reset();
15972 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15973 bgc_t_join.restart();
15975 if (heap_number == 0)
15978 bgc_threads_sync_event.Set();
15982 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15983 dprintf (2, ("bgc_threads_sync_event is signalled"));
15987 #endif //MULTIPLE_HEAPS
15989 //fix the allocation area so verify_heap can proceed.
15990 fix_allocation_contexts (FALSE);
15992 #endif //BACKGROUND_GC
15994 #ifdef BACKGROUND_GC
15995 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15996 #ifdef FEATURE_EVENT_TRACE
15997 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15999 GCToEEInterface::DiagWalkBGCSurvivors(__this);
16001 #ifdef MULTIPLE_HEAPS
16002 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
16003 if (bgc_t_join.joined())
16005 bgc_t_join.restart();
16007 #endif // MULTIPLE_HEAPS
16009 #endif // FEATURE_EVENT_TRACE
16010 #endif //BACKGROUND_GC
16013 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
16014 verify_heap (FALSE);
16015 #endif // VERIFY_HEAP
16017 #ifdef BACKGROUND_GC
16018 if (settings.concurrent)
16020 repair_allocation_contexts (TRUE);
16022 #ifdef MULTIPLE_HEAPS
16023 bgc_t_join.join(this, gc_join_restart_ee_verify);
16024 if (bgc_t_join.joined())
16026 bgc_threads_sync_event.Reset();
16028 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
16029 bgc_t_join.restart();
16031 if (heap_number == 0)
16034 bgc_threads_sync_event.Set();
16038 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16039 dprintf (2, ("bgc_threads_sync_event is signalled"));
16043 #endif //MULTIPLE_HEAPS
16045 disable_preemptive (cooperative_mode);
16047 #endif //BACKGROUND_GC
16049 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
16051 #ifdef MULTIPLE_HEAPS
16052 if (!settings.concurrent)
16054 gc_t_join.join(this, gc_join_done);
16055 if (gc_t_join.joined ())
16057 gc_heap::internal_gc_done = false;
16059 //equalize the new desired size of the generations
16060 int limit = settings.condemned_generation;
16061 if (limit == max_generation)
16063 limit = max_generation+1;
16065 for (int gen = 0; gen <= limit; gen++)
16067 size_t total_desired = 0;
16069 for (int i = 0; i < gc_heap::n_heaps; i++)
16071 gc_heap* hp = gc_heap::g_heaps[i];
16072 dynamic_data* dd = hp->dynamic_data_of (gen);
16073 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
16074 if (temp_total_desired < total_desired)
16077 total_desired = (size_t)MAX_PTR;
16080 total_desired = temp_total_desired;
16083 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
16084 get_alignment_constant ((gen != (max_generation+1))));
16088 #if 1 //subsumed by the linear allocation model
16089 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16090 // apply some smoothing.
16091 static size_t smoothed_desired_per_heap = 0;
16092 size_t smoothing = 3; // exponential smoothing factor
16093 if (smoothing > VolatileLoad(&settings.gc_index))
16094 smoothing = VolatileLoad(&settings.gc_index);
16095 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
16096 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
16097 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
16100 if (!heap_hard_limit)
16102 // if desired_per_heap is close to min_gc_size, trim it
16103 // down to min_gc_size to stay in the cache
16104 gc_heap* hp = gc_heap::g_heaps[0];
16105 dynamic_data* dd = hp->dynamic_data_of (gen);
16106 size_t min_gc_size = dd_min_size(dd);
16107 // if min GC size larger than true on die cache, then don't bother
16108 // limiting the desired size
16109 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
16110 desired_per_heap <= 2*min_gc_size)
16112 desired_per_heap = min_gc_size;
16116 desired_per_heap = joined_youngest_desired (desired_per_heap);
16117 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
16119 gc_data_global.final_youngest_desired = desired_per_heap;
16121 #if 1 //subsumed by the linear allocation model
16122 if (gen == (max_generation + 1))
16124 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16125 // apply some smoothing.
16126 static size_t smoothed_desired_per_heap_loh = 0;
16127 size_t smoothing = 3; // exponential smoothing factor
16128 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
16129 if (smoothing > loh_count)
16130 smoothing = loh_count;
16131 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
16132 dprintf (2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
16133 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
16136 for (int i = 0; i < gc_heap::n_heaps; i++)
16138 gc_heap* hp = gc_heap::g_heaps[i];
16139 dynamic_data* dd = hp->dynamic_data_of (gen);
16140 dd_desired_allocation (dd) = desired_per_heap;
16141 dd_gc_new_allocation (dd) = desired_per_heap;
16142 dd_new_allocation (dd) = desired_per_heap;
16146 hp->fgn_last_alloc = desired_per_heap;
16151 #ifdef FEATURE_LOH_COMPACTION
16152 BOOL all_heaps_compacted_p = TRUE;
16153 #endif //FEATURE_LOH_COMPACTION
16154 for (int i = 0; i < gc_heap::n_heaps; i++)
16156 gc_heap* hp = gc_heap::g_heaps[i];
16157 hp->decommit_ephemeral_segment_pages();
16158 hp->rearrange_large_heap_segments();
16159 #ifdef FEATURE_LOH_COMPACTION
16160 all_heaps_compacted_p &= hp->loh_compacted_p;
16161 #endif //FEATURE_LOH_COMPACTION
16164 #ifdef FEATURE_LOH_COMPACTION
16165 check_loh_compact_mode (all_heaps_compacted_p);
16166 #endif //FEATURE_LOH_COMPACTION
16169 pm_full_gc_init_or_clear();
16171 gc_t_join.restart();
16173 alloc_context_count = 0;
16174 heap_select::mark_heap (heap_number);
16178 gc_data_global.final_youngest_desired =
16179 dd_desired_allocation (dynamic_data_of (0));
16181 check_loh_compact_mode (loh_compacted_p);
16183 decommit_ephemeral_segment_pages();
16186 if (!(settings.concurrent))
16188 rearrange_large_heap_segments();
16192 pm_full_gc_init_or_clear();
16194 #ifdef BACKGROUND_GC
16195 recover_bgc_settings();
16196 #endif //BACKGROUND_GC
16197 #endif //MULTIPLE_HEAPS
16200 void gc_heap::save_data_for_no_gc()
16202 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
16203 #ifdef MULTIPLE_HEAPS
16204 // This is to affect heap balancing.
16205 for (int i = 0; i < n_heaps; i++)
16207 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
16208 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
16209 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
16210 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
16212 #endif //MULTIPLE_HEAPS
16215 void gc_heap::restore_data_for_no_gc()
16217 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
16218 #ifdef MULTIPLE_HEAPS
16219 for (int i = 0; i < n_heaps; i++)
16221 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
16222 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
16224 #endif //MULTIPLE_HEAPS
16227 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
16228 BOOL loh_size_known,
16230 BOOL disallow_full_blocking)
16232 if (current_no_gc_region_info.started)
16234 return start_no_gc_in_progress;
16237 start_no_gc_region_status status = start_no_gc_success;
16239 save_data_for_no_gc();
16240 settings.pause_mode = pause_no_gc;
16241 current_no_gc_region_info.start_status = start_no_gc_success;
16243 uint64_t allocation_no_gc_loh = 0;
16244 uint64_t allocation_no_gc_soh = 0;
16245 assert(total_size != 0);
16246 if (loh_size_known)
16248 assert(loh_size != 0);
16249 assert(loh_size <= total_size);
16250 allocation_no_gc_loh = loh_size;
16251 allocation_no_gc_soh = total_size - loh_size;
16255 allocation_no_gc_soh = total_size;
16256 allocation_no_gc_loh = total_size;
16259 int soh_align_const = get_alignment_constant (TRUE);
16260 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
16261 size_t size_per_heap = 0;
16262 const double scale_factor = 1.05;
16265 #ifdef MULTIPLE_HEAPS
16266 num_heaps = n_heaps;
16267 #endif // MULTIPLE_HEAPS
16269 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
16271 // In theory, the upper limit here is the physical memory of the machine, not
16272 // SIZE_T_MAX. This is not true today because total_physical_mem can be
16273 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
16274 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
16275 // more freely between branches, it would be good to clean this up to use
16276 // total_physical_mem instead of SIZE_T_MAX.
16277 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
16278 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
16279 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
16280 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
16282 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
16283 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
16285 status = start_no_gc_too_large;
16289 if (allocation_no_gc_soh > 0)
16291 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
16292 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
16295 if (allocation_no_gc_loh > 0)
16297 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
16298 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
16301 if (disallow_full_blocking)
16302 current_no_gc_region_info.minimal_gc_p = TRUE;
16304 if (allocation_no_gc_soh != 0)
16306 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
16307 size_per_heap = current_no_gc_region_info.soh_allocation_size;
16308 #ifdef MULTIPLE_HEAPS
16309 size_per_heap /= n_heaps;
16310 for (int i = 0; i < n_heaps; i++)
16312 // due to heap balancing we need to allow some room before we even look to balance to another heap.
16313 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
16315 #else //MULTIPLE_HEAPS
16316 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
16317 #endif //MULTIPLE_HEAPS
16320 if (allocation_no_gc_loh != 0)
16322 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
16323 size_per_heap = current_no_gc_region_info.loh_allocation_size;
16324 #ifdef MULTIPLE_HEAPS
16325 size_per_heap /= n_heaps;
16326 for (int i = 0; i < n_heaps; i++)
16327 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16328 #else //MULTIPLE_HEAPS
16329 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16330 #endif //MULTIPLE_HEAPS
16334 if (status != start_no_gc_success)
16335 restore_data_for_no_gc();
16339 void gc_heap::handle_failure_for_no_gc()
16341 gc_heap::restore_data_for_no_gc();
16342 // sets current_no_gc_region_info.started to FALSE here.
16343 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16346 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
16348 return current_no_gc_region_info.start_status;
16351 void gc_heap::record_gcs_during_no_gc()
16353 if (current_no_gc_region_info.started)
16355 current_no_gc_region_info.num_gcs++;
16356 if (is_induced (settings.reason))
16357 current_no_gc_region_info.num_gcs_induced++;
16361 BOOL gc_heap::find_loh_free_for_no_gc()
16363 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
16364 size_t sz_list = loh_allocator->first_bucket_size();
16365 size_t size = loh_allocation_no_gc;
16366 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
16368 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
16370 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
16373 size_t free_list_size = unused_array_size(free_list);
16375 if (free_list_size > loh_allocation_no_gc)
16377 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
16381 free_list = free_list_slot (free_list);
16384 sz_list = sz_list * 2;
16390 BOOL gc_heap::find_loh_space_for_no_gc()
16392 saved_loh_segment_no_gc = 0;
16394 if (find_loh_free_for_no_gc())
16397 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16401 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16402 if (remaining >= loh_allocation_no_gc)
16404 saved_loh_segment_no_gc = seg;
16407 seg = heap_segment_next (seg);
16410 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16412 // If no full GC is allowed, we try to get a new seg right away.
16413 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16414 #ifdef MULTIPLE_HEAPS
16416 #endif //MULTIPLE_HEAPS
16420 return (saved_loh_segment_no_gc != 0);
16423 BOOL gc_heap::loh_allocated_for_no_gc()
16425 if (!saved_loh_segment_no_gc)
16428 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16431 if (seg == saved_loh_segment_no_gc)
16435 seg = heap_segment_next (seg);
16441 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16443 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16444 assert (end_committed <= heap_segment_reserved (seg));
16445 return (grow_heap_segment (seg, end_committed));
16448 void gc_heap::thread_no_gc_loh_segments()
16450 #ifdef MULTIPLE_HEAPS
16451 for (int i = 0; i < n_heaps; i++)
16453 gc_heap* hp = g_heaps[i];
16454 if (hp->loh_allocated_for_no_gc())
16456 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16457 hp->saved_loh_segment_no_gc = 0;
16460 #else //MULTIPLE_HEAPS
16461 if (loh_allocated_for_no_gc())
16463 thread_loh_segment (saved_loh_segment_no_gc);
16464 saved_loh_segment_no_gc = 0;
16466 #endif //MULTIPLE_HEAPS
16469 void gc_heap::set_loh_allocations_for_no_gc()
16471 if (current_no_gc_region_info.loh_allocation_size != 0)
16473 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16474 dd_new_allocation (dd) = loh_allocation_no_gc;
16475 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16479 void gc_heap::set_soh_allocations_for_no_gc()
16481 if (current_no_gc_region_info.soh_allocation_size != 0)
16483 dynamic_data* dd = dynamic_data_of (0);
16484 dd_new_allocation (dd) = soh_allocation_no_gc;
16485 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16486 #ifdef MULTIPLE_HEAPS
16487 alloc_context_count = 0;
16488 #endif //MULTIPLE_HEAPS
16492 void gc_heap::set_allocations_for_no_gc()
16494 #ifdef MULTIPLE_HEAPS
16495 for (int i = 0; i < n_heaps; i++)
16497 gc_heap* hp = g_heaps[i];
16498 hp->set_loh_allocations_for_no_gc();
16499 hp->set_soh_allocations_for_no_gc();
16501 #else //MULTIPLE_HEAPS
16502 set_loh_allocations_for_no_gc();
16503 set_soh_allocations_for_no_gc();
16504 #endif //MULTIPLE_HEAPS
16507 BOOL gc_heap::should_proceed_for_no_gc()
16509 BOOL gc_requested = FALSE;
16510 BOOL loh_full_gc_requested = FALSE;
16511 BOOL soh_full_gc_requested = FALSE;
16512 BOOL no_gc_requested = FALSE;
16513 BOOL get_new_loh_segments = FALSE;
16515 if (current_no_gc_region_info.soh_allocation_size)
16517 #ifdef MULTIPLE_HEAPS
16518 for (int i = 0; i < n_heaps; i++)
16520 gc_heap* hp = g_heaps[i];
16521 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16523 gc_requested = TRUE;
16527 #else //MULTIPLE_HEAPS
16528 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16529 gc_requested = TRUE;
16530 #endif //MULTIPLE_HEAPS
16534 #ifdef MULTIPLE_HEAPS
16535 for (int i = 0; i < n_heaps; i++)
16537 gc_heap* hp = g_heaps[i];
16538 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16540 soh_full_gc_requested = TRUE;
16544 #else //MULTIPLE_HEAPS
16545 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16546 soh_full_gc_requested = TRUE;
16547 #endif //MULTIPLE_HEAPS
16551 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16553 soh_full_gc_requested = TRUE;
16556 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16558 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16560 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16564 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16566 // Check to see if we have enough reserved space.
16567 #ifdef MULTIPLE_HEAPS
16568 for (int i = 0; i < n_heaps; i++)
16570 gc_heap* hp = g_heaps[i];
16571 if (!hp->find_loh_space_for_no_gc())
16573 loh_full_gc_requested = TRUE;
16577 #else //MULTIPLE_HEAPS
16578 if (!find_loh_space_for_no_gc())
16579 loh_full_gc_requested = TRUE;
16580 #endif //MULTIPLE_HEAPS
16582 // Check to see if we have committed space.
16583 if (!loh_full_gc_requested)
16585 #ifdef MULTIPLE_HEAPS
16586 for (int i = 0; i < n_heaps; i++)
16588 gc_heap* hp = g_heaps[i];
16589 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16591 loh_full_gc_requested = TRUE;
16595 #else //MULTIPLE_HEAPS
16596 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16597 loh_full_gc_requested = TRUE;
16598 #endif //MULTIPLE_HEAPS
16602 if (loh_full_gc_requested || soh_full_gc_requested)
16604 if (current_no_gc_region_info.minimal_gc_p)
16605 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16608 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16610 if (current_no_gc_region_info.start_status == start_no_gc_success)
16612 if (no_gc_requested)
16613 set_allocations_for_no_gc();
16618 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16622 // We are done with starting the no_gc_region.
16623 current_no_gc_region_info.started = TRUE;
16628 end_no_gc_region_status gc_heap::end_no_gc_region()
16630 dprintf (1, ("end no gc called"));
16632 end_no_gc_region_status status = end_no_gc_success;
16634 if (!(current_no_gc_region_info.started))
16635 status = end_no_gc_not_in_progress;
16636 if (current_no_gc_region_info.num_gcs_induced)
16637 status = end_no_gc_induced;
16638 else if (current_no_gc_region_info.num_gcs)
16639 status = end_no_gc_alloc_exceeded;
16641 if (settings.pause_mode == pause_no_gc)
16642 restore_data_for_no_gc();
16644 // sets current_no_gc_region_info.started to FALSE here.
16645 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16651 void gc_heap::update_collection_counts ()
16653 dynamic_data* dd0 = dynamic_data_of (0);
16654 dd_gc_clock (dd0) += 1;
16656 size_t now = GetHighPrecisionTimeStamp();
16658 for (int i = 0; i <= settings.condemned_generation;i++)
16660 dynamic_data* dd = dynamic_data_of (i);
16661 dd_collection_count (dd)++;
16662 //this is needed by the linear allocation model
16663 if (i == max_generation)
16664 dd_collection_count (dynamic_data_of (max_generation+1))++;
16665 dd_gc_clock (dd) = dd_gc_clock (dd0);
16666 dd_time_clock (dd) = now;
16670 BOOL gc_heap::expand_soh_with_minimal_gc()
16672 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16675 heap_segment* new_seg = soh_get_segment_to_expand();
16678 if (g_gc_card_table != card_table)
16679 copy_brick_card_table();
16681 settings.promotion = TRUE;
16682 settings.demotion = FALSE;
16683 ephemeral_promotion = TRUE;
16684 int condemned_gen_number = max_generation - 1;
16686 generation* gen = 0;
16687 int align_const = get_alignment_constant (TRUE);
16689 for (int i = 0; i <= condemned_gen_number; i++)
16691 gen = generation_of (i);
16692 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16693 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16696 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16697 // and need to make sure that there are no left over bricks from the previous GCs for the space
16698 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16699 // ephemeral GCs later.
16700 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16701 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16707 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16708 generation_allocation_start (generation_of (max_generation - 1)));
16709 heap_segment_next (ephemeral_heap_segment) = new_seg;
16710 ephemeral_heap_segment = new_seg;
16711 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16713 for (int i = condemned_gen_number; i >= 0; i--)
16715 gen = generation_of (i);
16716 size_t gen_start_size = Align (min_obj_size);
16717 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16718 generation_plan_allocation_start (gen) = start;
16719 generation_plan_allocation_start_size (gen) = gen_start_size;
16720 start += gen_start_size;
16722 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16723 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16725 fix_generation_bounds (condemned_gen_number, generation_of (0));
16727 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16728 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16730 adjust_ephemeral_limits();
16737 // Only to be done on the thread that calls restart in a join for server GC
16738 // and reset the oom status per heap.
16739 void gc_heap::check_and_set_no_gc_oom()
16741 #ifdef MULTIPLE_HEAPS
16742 for (int i = 0; i < n_heaps; i++)
16744 gc_heap* hp = g_heaps[i];
16745 if (hp->no_gc_oom_p)
16747 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16748 hp->no_gc_oom_p = false;
16754 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16755 no_gc_oom_p = false;
16757 #endif //MULTIPLE_HEAPS
16760 void gc_heap::allocate_for_no_gc_after_gc()
16762 if (current_no_gc_region_info.minimal_gc_p)
16763 repair_allocation_contexts (TRUE);
16765 no_gc_oom_p = false;
16767 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16769 if (current_no_gc_region_info.soh_allocation_size != 0)
16771 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16772 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16774 no_gc_oom_p = true;
16777 #ifdef MULTIPLE_HEAPS
16778 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16779 if (gc_t_join.joined())
16781 #endif //MULTIPLE_HEAPS
16783 check_and_set_no_gc_oom();
16785 #ifdef MULTIPLE_HEAPS
16786 gc_t_join.restart();
16788 #endif //MULTIPLE_HEAPS
16791 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16792 !(current_no_gc_region_info.minimal_gc_p) &&
16793 (current_no_gc_region_info.loh_allocation_size != 0))
16795 gc_policy = policy_compact;
16796 saved_loh_segment_no_gc = 0;
16798 if (!find_loh_free_for_no_gc())
16800 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16801 BOOL found_seg_p = FALSE;
16804 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16806 found_seg_p = TRUE;
16807 if (!commit_loh_for_no_gc (seg))
16809 no_gc_oom_p = true;
16813 seg = heap_segment_next (seg);
16817 gc_policy = policy_expand;
16820 #ifdef MULTIPLE_HEAPS
16821 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16822 if (gc_t_join.joined())
16824 check_and_set_no_gc_oom();
16826 if (current_no_gc_region_info.start_status == start_no_gc_success)
16828 for (int i = 0; i < n_heaps; i++)
16830 gc_heap* hp = g_heaps[i];
16831 if (hp->gc_policy == policy_expand)
16833 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16834 if (!(hp->saved_loh_segment_no_gc))
16836 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16843 gc_t_join.restart();
16845 #else //MULTIPLE_HEAPS
16846 check_and_set_no_gc_oom();
16848 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16850 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16851 if (!saved_loh_segment_no_gc)
16852 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16854 #endif //MULTIPLE_HEAPS
16856 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16858 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16860 no_gc_oom_p = true;
16866 #ifdef MULTIPLE_HEAPS
16867 gc_t_join.join(this, gc_join_final_no_gc);
16868 if (gc_t_join.joined())
16870 #endif //MULTIPLE_HEAPS
16872 check_and_set_no_gc_oom();
16874 if (current_no_gc_region_info.start_status == start_no_gc_success)
16876 set_allocations_for_no_gc();
16877 current_no_gc_region_info.started = TRUE;
16880 #ifdef MULTIPLE_HEAPS
16881 gc_t_join.restart();
16883 #endif //MULTIPLE_HEAPS
16886 void gc_heap::init_records()
16888 // An option is to move this to be after we figure out which gen to condemn so we don't
16889 // need to clear some generations' data 'cause we know they don't change, but that also means
16890 // we can't simply call memset here.
16891 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16892 gc_data_per_heap.heap_index = heap_number;
16893 if (heap_number == 0)
16894 memset (&gc_data_global, 0, sizeof (gc_data_global));
16896 #ifdef GC_CONFIG_DRIVEN
16897 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16898 #endif //GC_CONFIG_DRIVEN
16899 memset (&fgm_result, 0, sizeof (fgm_result));
16901 for (int i = 0; i <= (max_generation + 1); i++)
16903 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16904 generation* gen = generation_of (i);
16905 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16906 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16909 sufficient_gen0_space_p = FALSE;
16911 #ifdef MULTIPLE_HEAPS
16912 gen0_allocated_after_gc_p = false;
16913 #endif //MULTIPLE_HEAPS
16915 #if defined (_DEBUG) && defined (VERIFY_HEAP)
16916 verify_pinned_queue_p = FALSE;
16917 #endif // _DEBUG && VERIFY_HEAP
16920 void gc_heap::pm_full_gc_init_or_clear()
16922 // This means the next GC will be a full blocking GC and we need to init.
16923 if (settings.condemned_generation == (max_generation - 1))
16925 if (pm_trigger_full_gc)
16927 #ifdef MULTIPLE_HEAPS
16929 #endif //MULTIPLE_HEAPS
16930 dprintf (GTC_LOG, ("init for PM triggered full GC"));
16931 uint32_t saved_entry_memory_load = settings.entry_memory_load;
16932 settings.init_mechanisms();
16933 settings.reason = reason_pm_full_gc;
16934 settings.condemned_generation = max_generation;
16935 settings.entry_memory_load = saved_entry_memory_load;
16936 // Can't assert this since we only check at the end of gen2 GCs,
16937 // during gen1 the memory load could have already dropped.
16938 // Although arguably we should just turn off PM then...
16939 //assert (settings.entry_memory_load >= high_memory_load_th);
16940 assert (settings.entry_memory_load > 0);
16941 settings.gc_index += 1;
16945 // This means we are in the progress of a full blocking GC triggered by
16947 else if (settings.reason == reason_pm_full_gc)
16949 assert (settings.condemned_generation == max_generation);
16950 assert (pm_trigger_full_gc);
16951 pm_trigger_full_gc = false;
16953 dprintf (GTC_LOG, ("PM triggered full GC done"));
16957 void gc_heap::garbage_collect_pm_full_gc()
16959 assert (settings.condemned_generation == max_generation);
16960 assert (settings.reason == reason_pm_full_gc);
16961 assert (!settings.concurrent);
16965 void gc_heap::garbage_collect (int n)
16967 //reset the number of alloc contexts
16968 alloc_contexts_used = 0;
16970 fix_allocation_contexts (TRUE);
16971 #ifdef MULTIPLE_HEAPS
16973 gc_t_join.start_ts(this);
16974 #endif //JOIN_STATS
16975 clear_gen0_bricks();
16976 #endif //MULTIPLE_HEAPS
16978 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16980 #ifdef MULTIPLE_HEAPS
16981 gc_t_join.join(this, gc_join_minimal_gc);
16982 if (gc_t_join.joined())
16984 #endif //MULTIPLE_HEAPS
16986 #ifdef MULTIPLE_HEAPS
16987 // this is serialized because we need to get a segment
16988 for (int i = 0; i < n_heaps; i++)
16990 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16991 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16994 if (!expand_soh_with_minimal_gc())
16995 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16996 #endif //MULTIPLE_HEAPS
16998 update_collection_counts_for_no_gc();
17000 #ifdef MULTIPLE_HEAPS
17001 gc_t_join.restart();
17003 #endif //MULTIPLE_HEAPS
17010 settings.reason = gc_trigger_reason;
17011 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
17012 num_pinned_objects = 0;
17013 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
17016 if (settings.reason == reason_gcstress)
17018 settings.reason = reason_induced;
17019 settings.stress_induced = TRUE;
17021 #endif // STRESS_HEAP
17023 #ifdef MULTIPLE_HEAPS
17024 //align all heaps on the max generation to condemn
17025 dprintf (3, ("Joining for max generation to condemn"));
17026 condemned_generation_num = generation_to_condemn (n,
17027 &blocking_collection,
17028 &elevation_requested,
17030 gc_t_join.join(this, gc_join_generation_determined);
17031 if (gc_t_join.joined())
17032 #endif //MULTIPLE_HEAPS
17034 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
17035 //delete old slots from the segment table
17036 seg_table->delete_old_slots();
17037 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
17039 #ifdef MULTIPLE_HEAPS
17040 for (int i = 0; i < n_heaps; i++)
17042 gc_heap* hp = g_heaps[i];
17043 // check for card table growth
17044 if (g_gc_card_table != hp->card_table)
17045 hp->copy_brick_card_table();
17047 hp->rearrange_large_heap_segments();
17048 #ifdef BACKGROUND_GC
17049 hp->background_delay_delete_loh_segments();
17050 if (!recursive_gc_sync::background_running_p())
17051 hp->rearrange_small_heap_segments();
17052 #endif //BACKGROUND_GC
17054 #else //MULTIPLE_HEAPS
17055 if (g_gc_card_table != card_table)
17056 copy_brick_card_table();
17058 rearrange_large_heap_segments();
17059 #ifdef BACKGROUND_GC
17060 background_delay_delete_loh_segments();
17061 if (!recursive_gc_sync::background_running_p())
17062 rearrange_small_heap_segments();
17063 #endif //BACKGROUND_GC
17064 #endif //MULTIPLE_HEAPS
17066 BOOL should_evaluate_elevation = TRUE;
17067 BOOL should_do_blocking_collection = FALSE;
17069 #ifdef MULTIPLE_HEAPS
17070 int gen_max = condemned_generation_num;
17071 for (int i = 0; i < n_heaps; i++)
17073 if (gen_max < g_heaps[i]->condemned_generation_num)
17074 gen_max = g_heaps[i]->condemned_generation_num;
17075 if (should_evaluate_elevation && !(g_heaps[i]->elevation_requested))
17076 should_evaluate_elevation = FALSE;
17077 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
17078 should_do_blocking_collection = TRUE;
17081 settings.condemned_generation = gen_max;
17082 #else //MULTIPLE_HEAPS
17083 settings.condemned_generation = generation_to_condemn (n,
17084 &blocking_collection,
17085 &elevation_requested,
17087 should_evaluate_elevation = elevation_requested;
17088 should_do_blocking_collection = blocking_collection;
17089 #endif //MULTIPLE_HEAPS
17091 settings.condemned_generation = joined_generation_to_condemn (
17092 should_evaluate_elevation,
17094 settings.condemned_generation,
17095 &should_do_blocking_collection
17099 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
17100 "condemned generation num: %d\n", settings.condemned_generation);
17102 record_gcs_during_no_gc();
17104 if (settings.condemned_generation > 1)
17105 settings.promotion = TRUE;
17107 #ifdef HEAP_ANALYZE
17108 // At this point we've decided what generation is condemned
17109 // See if we've been requested to analyze survivors after the mark phase
17110 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
17112 heap_analyze_enabled = TRUE;
17114 #endif // HEAP_ANALYZE
17116 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
17118 #ifdef BACKGROUND_GC
17119 if ((settings.condemned_generation == max_generation) &&
17120 (recursive_gc_sync::background_running_p()))
17122 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
17123 // because we have to collect 0 and 1 properly
17124 // in particular, the allocation contexts are gone.
17125 // For now, it is simpler to collect max_generation-1
17126 settings.condemned_generation = max_generation - 1;
17127 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
17130 if ((settings.condemned_generation == max_generation) &&
17131 (should_do_blocking_collection == FALSE) &&
17132 gc_can_use_concurrent &&
17133 !temp_disable_concurrent_p &&
17134 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
17136 keep_bgc_threads_p = TRUE;
17137 c_write (settings.concurrent, TRUE);
17139 #endif //BACKGROUND_GC
17141 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
17143 // Call the EE for start of GC work
17144 // just one thread for MP GC
17145 GCToEEInterface::GcStartWork (settings.condemned_generation,
17148 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
17149 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
17150 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
17154 #ifdef MULTIPLE_HEAPS
17155 gc_start_event.Reset();
17156 //start all threads on the roots.
17157 dprintf(3, ("Starting all gc threads for gc"));
17158 gc_t_join.restart();
17159 #endif //MULTIPLE_HEAPS
17162 descr_generations (TRUE);
17165 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
17166 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
17168 verify_heap (TRUE);
17170 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
17171 checkGCWriteBarrier();
17173 #endif // VERIFY_HEAP
17175 #ifdef BACKGROUND_GC
17176 if (settings.concurrent)
17178 // We need to save the settings because we'll need to restore it after each FGC.
17179 assert (settings.condemned_generation == max_generation);
17180 settings.compaction = FALSE;
17181 saved_bgc_settings = settings;
17183 #ifdef MULTIPLE_HEAPS
17184 if (heap_number == 0)
17186 for (int i = 0; i < n_heaps; i++)
17188 prepare_bgc_thread (g_heaps[i]);
17190 dprintf (2, ("setting bgc_threads_sync_event"));
17191 bgc_threads_sync_event.Set();
17195 bgc_threads_sync_event.Wait(INFINITE, FALSE);
17196 dprintf (2, ("bgc_threads_sync_event is signalled"));
17199 prepare_bgc_thread(0);
17200 #endif //MULTIPLE_HEAPS
17202 #ifdef MULTIPLE_HEAPS
17203 gc_t_join.join(this, gc_join_start_bgc);
17204 if (gc_t_join.joined())
17205 #endif //MULTIPLE_HEAPS
17207 do_concurrent_p = TRUE;
17208 do_ephemeral_gc_p = FALSE;
17209 #ifdef MULTIPLE_HEAPS
17210 dprintf(2, ("Joined to perform a background GC"));
17212 for (int i = 0; i < n_heaps; i++)
17214 gc_heap* hp = g_heaps[i];
17215 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
17217 do_concurrent_p = FALSE;
17222 hp->background_saved_lowest_address = hp->lowest_address;
17223 hp->background_saved_highest_address = hp->highest_address;
17227 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
17228 if (do_concurrent_p)
17230 background_saved_lowest_address = lowest_address;
17231 background_saved_highest_address = highest_address;
17233 #endif //MULTIPLE_HEAPS
17235 if (do_concurrent_p)
17237 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17238 SoftwareWriteWatch::EnableForGCHeap();
17239 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17241 #ifdef MULTIPLE_HEAPS
17242 for (int i = 0; i < n_heaps; i++)
17243 g_heaps[i]->current_bgc_state = bgc_initialized;
17245 current_bgc_state = bgc_initialized;
17246 #endif //MULTIPLE_HEAPS
17248 int gen = check_for_ephemeral_alloc();
17249 // always do a gen1 GC before we start BGC.
17250 // This is temporary for testing purpose.
17251 //int gen = max_generation - 1;
17252 dont_restart_ee_p = TRUE;
17255 // If we decide to not do a GC before the BGC we need to
17256 // restore the gen0 alloc context.
17257 #ifdef MULTIPLE_HEAPS
17258 for (int i = 0; i < n_heaps; i++)
17260 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
17261 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
17264 generation_allocation_pointer (youngest_generation) = 0;
17265 generation_allocation_limit (youngest_generation) = 0;
17266 #endif //MULTIPLE_HEAPS
17270 do_ephemeral_gc_p = TRUE;
17272 settings.init_mechanisms();
17273 settings.condemned_generation = gen;
17274 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
17277 // TODO BACKGROUND_GC need to add the profiling stuff here.
17278 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
17281 //clear the cards so they don't bleed in gen 1 during collection
17282 // shouldn't this always be done at the beginning of any GC?
17283 //clear_card_for_addresses (
17284 // generation_allocation_start (generation_of (0)),
17285 // heap_segment_allocated (ephemeral_heap_segment));
17287 if (!do_ephemeral_gc_p)
17289 do_background_gc();
17294 settings.compaction = TRUE;
17295 c_write (settings.concurrent, FALSE);
17298 #ifdef MULTIPLE_HEAPS
17299 gc_t_join.restart();
17300 #endif //MULTIPLE_HEAPS
17303 if (do_concurrent_p)
17305 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
17306 // global data is only calculated at the end of the GC so we don't need to worry about
17307 // FGCs overwriting it.
17308 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
17309 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
17311 if (do_ephemeral_gc_p)
17313 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
17315 gen_to_condemn_reasons.init();
17316 gen_to_condemn_reasons.set_condition (gen_before_bgc);
17317 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
17319 #ifdef MULTIPLE_HEAPS
17320 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
17321 if (gc_t_join.joined())
17322 #endif //MULTIPLE_HEAPS
17324 #ifdef MULTIPLE_HEAPS
17326 #endif //MULTIPLE_HEAPS
17327 settings = saved_bgc_settings;
17328 assert (settings.concurrent);
17330 do_background_gc();
17332 #ifdef MULTIPLE_HEAPS
17333 gc_t_join.restart();
17334 #endif //MULTIPLE_HEAPS
17340 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
17345 #endif //BACKGROUND_GC
17349 #ifndef MULTIPLE_HEAPS
17350 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
17351 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
17352 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
17353 #endif //MULTIPLE_HEAPS
17356 if (settings.pause_mode == pause_no_gc)
17357 allocate_for_no_gc_after_gc();
17361 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
17364 size_t& gc_heap::promoted_bytes(int thread)
17366 #ifdef MULTIPLE_HEAPS
17367 return g_promoted [thread*16];
17368 #else //MULTIPLE_HEAPS
17369 UNREFERENCED_PARAMETER(thread);
17371 #endif //MULTIPLE_HEAPS
17374 #ifdef INTERIOR_POINTERS
17375 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
17377 #ifdef SEG_MAPPING_TABLE
17378 heap_segment* seg = seg_mapping_table_segment_of (interior);
17381 if (small_segment_only_p && heap_segment_loh_p (seg))
17385 #else //SEG_MAPPING_TABLE
17386 #ifdef MULTIPLE_HEAPS
17387 for (int i = 0; i < gc_heap::n_heaps; i++)
17389 gc_heap* h = gc_heap::g_heaps [i];
17390 hs = h->find_segment_per_heap (o, small_segment_only_p);
17398 gc_heap* h = pGenGCHeap;
17399 hs = h->find_segment_per_heap (o, small_segment_only_p);
17401 #endif //MULTIPLE_HEAPS
17402 #endif //SEG_MAPPING_TABLE
17405 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
17407 #ifdef SEG_MAPPING_TABLE
17408 return find_segment (interior, small_segment_only_p);
17409 #else //SEG_MAPPING_TABLE
17410 if (in_range_for_segment (interior, ephemeral_heap_segment))
17412 return ephemeral_heap_segment;
17416 heap_segment* found_seg = 0;
17419 heap_segment* seg = generation_start_segment (generation_of (max_generation));
17422 if (in_range_for_segment (interior, seg))
17425 goto end_find_segment;
17428 } while ((seg = heap_segment_next (seg)) != 0);
17430 if (!small_segment_only_p)
17432 #ifdef BACKGROUND_GC
17434 ptrdiff_t delta = 0;
17435 heap_segment* seg = segment_of (interior, delta);
17436 if (seg && in_range_for_segment (interior, seg))
17440 goto end_find_segment;
17442 #else //BACKGROUND_GC
17443 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17446 if (in_range_for_segment(interior, seg))
17449 goto end_find_segment;
17452 } while ((seg = heap_segment_next (seg)) != 0);
17453 #endif //BACKGROUND_GC
17459 #endif //SEG_MAPPING_TABLE
17461 #endif //INTERIOR_POINTERS
17463 #if !defined(_DEBUG) && !defined(__GNUC__)
17464 inline // This causes link errors if global optimization is off
17465 #endif //!_DEBUG && !__GNUC__
17466 gc_heap* gc_heap::heap_of (uint8_t* o)
17468 #ifdef MULTIPLE_HEAPS
17470 return g_heaps [0];
17471 #ifdef SEG_MAPPING_TABLE
17472 gc_heap* hp = seg_mapping_table_heap_of (o);
17473 return (hp ? hp : g_heaps[0]);
17474 #else //SEG_MAPPING_TABLE
17475 ptrdiff_t delta = 0;
17476 heap_segment* seg = segment_of (o, delta);
17477 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17478 #endif //SEG_MAPPING_TABLE
17479 #else //MULTIPLE_HEAPS
17480 UNREFERENCED_PARAMETER(o);
17482 #endif //MULTIPLE_HEAPS
17486 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17488 #ifdef MULTIPLE_HEAPS
17490 return g_heaps [0];
17491 #ifdef SEG_MAPPING_TABLE
17492 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17493 return (hp ? hp : g_heaps[0]);
17494 #else //SEG_MAPPING_TABLE
17495 ptrdiff_t delta = 0;
17496 heap_segment* seg = segment_of (o, delta);
17497 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17498 #endif //SEG_MAPPING_TABLE
17499 #else //MULTIPLE_HEAPS
17500 UNREFERENCED_PARAMETER(o);
17502 #endif //MULTIPLE_HEAPS
17505 #ifdef INTERIOR_POINTERS
17506 // will find all heap objects (large and small)
17507 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17509 if (!gen0_bricks_cleared)
17511 #ifdef MULTIPLE_HEAPS
17512 assert (!"Should have already been done in server GC");
17513 #endif //MULTIPLE_HEAPS
17514 gen0_bricks_cleared = TRUE;
17515 //initialize brick table for gen 0
17516 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17517 b < brick_of (align_on_brick
17518 (heap_segment_allocated (ephemeral_heap_segment)));
17524 #ifdef FFIND_OBJECT
17525 //indicate that in the future this needs to be done during allocation
17526 #ifdef MULTIPLE_HEAPS
17527 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17529 gen0_must_clear_bricks = FFIND_DECAY;
17530 #endif //MULTIPLE_HEAPS
17531 #endif //FFIND_OBJECT
17533 int brick_entry = get_brick_entry(brick_of (interior));
17534 if (brick_entry == 0)
17536 // this is a pointer to a large object
17537 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17539 #ifdef FEATURE_CONSERVATIVE_GC
17540 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17544 // If interior falls within the first free object at the beginning of a generation,
17545 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17546 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17547 #ifdef FEATURE_CONSERVATIVE_GC
17548 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17551 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17552 assert (interior < heap_segment_allocated (seg));
17554 uint8_t* o = heap_segment_mem (seg);
17555 while (o < heap_segment_allocated (seg))
17557 uint8_t* next_o = o + Align (size (o), align_const);
17558 assert (next_o > o);
17559 if ((o <= interior) && (interior < next_o))
17570 else if (interior >= low)
17572 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17575 #ifdef FEATURE_CONSERVATIVE_GC
17576 if (interior >= heap_segment_allocated (seg))
17579 assert (interior < heap_segment_allocated (seg));
17581 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17592 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17594 uint8_t* old_address = interior;
17595 if (!((old_address >= low) && (old_address < high)))
17598 size_t brick = brick_of (old_address);
17599 int brick_entry = brick_table [ brick ];
17600 if (brick_entry != 0)
17604 while (brick_entry < 0)
17606 brick = (brick + brick_entry);
17607 brick_entry = brick_table [ brick ];
17609 uint8_t* old_loc = old_address;
17610 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17612 if (node <= old_loc)
17617 brick_entry = brick_table [ brick ];
17623 //find the object by going along the plug
17625 while (o <= interior)
17627 uint8_t* next_o = o + Align (size (o));
17628 assert (next_o > o);
17629 if (next_o > interior)
17635 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17640 // this is a pointer to a large object
17641 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17644 assert (interior < heap_segment_allocated (seg));
17646 uint8_t* o = heap_segment_mem (seg);
17647 while (o < heap_segment_allocated (seg))
17649 uint8_t* next_o = o + Align (size (o));
17650 assert (next_o > o);
17651 if ((o < interior) && (interior < next_o))
17663 #else //INTERIOR_POINTERS
17665 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17669 #endif //INTERIOR_POINTERS
17671 #ifdef MULTIPLE_HEAPS
17674 #ifdef GC_CONFIG_DRIVEN
17675 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;}}
17676 #else //GC_CONFIG_DRIVEN
17677 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}}
17678 #endif //GC_CONFIG_DRIVEN
17680 #define m_boundary(o) {}
17683 #define m_boundary_fullgc(o) {}
17685 #else //MULTIPLE_HEAPS
17688 #ifdef GC_CONFIG_DRIVEN
17689 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17691 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17692 #endif //GC_CONFIG_DRIVEN
17694 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17697 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17699 #endif //MULTIPLE_HEAPS
17701 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17704 BOOL gc_heap::gc_mark1 (uint8_t* o)
17706 BOOL marked = !marked (o);
17708 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17713 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17715 BOOL marked = FALSE;
17716 if ((o >= low) && (o < high))
17717 marked = gc_mark1 (o);
17718 #ifdef MULTIPLE_HEAPS
17722 gc_heap* hp = heap_of_gc (o);
17724 if ((o >= hp->gc_low) && (o < hp->gc_high))
17725 marked = gc_mark1 (o);
17728 snoop_stat.objects_checked_count++;
17732 snoop_stat.objects_marked_count++;
17736 snoop_stat.zero_ref_count++;
17739 #endif //SNOOP_STATS
17740 #endif //MULTIPLE_HEAPS
17744 #ifdef BACKGROUND_GC
17747 BOOL gc_heap::background_marked (uint8_t* o)
17749 return mark_array_marked (o);
17752 BOOL gc_heap::background_mark1 (uint8_t* o)
17754 BOOL to_mark = !mark_array_marked (o);
17756 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17759 mark_array_set_marked (o);
17760 dprintf (4, ("n*%Ix*n", (size_t)o));
17767 // TODO: we could consider filtering out NULL's here instead of going to
17768 // look for it on other heaps
17770 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17772 BOOL marked = FALSE;
17773 if ((o >= low) && (o < high))
17774 marked = background_mark1 (o);
17775 #ifdef MULTIPLE_HEAPS
17779 gc_heap* hp = heap_of (o);
17781 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17782 marked = background_mark1 (o);
17784 #endif //MULTIPLE_HEAPS
17788 #endif //BACKGROUND_GC
17791 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17793 if (seg == ephemeral_heap_segment)
17796 return heap_segment_allocated (seg);
17799 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17800 #define ignore_start 0
17801 #define use_start 1
17803 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17805 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17806 CGCDescSeries* cur = map->GetHighestSeries(); \
17807 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17811 CGCDescSeries* last = map->GetLowestSeries(); \
17812 uint8_t** parm = 0; \
17815 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17816 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17817 uint8_t** ppstop = \
17818 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17819 if (!start_useful || (uint8_t*)ppstop > (start)) \
17821 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17822 while (parm < ppstop) \
17830 } while (cur >= last); \
17834 /* Handle the repeating case - array of valuetypes */ \
17835 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17836 if (start_useful && start > (uint8_t*)parm) \
17838 ptrdiff_t cs = mt->RawGetComponentSize(); \
17839 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17841 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17843 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17845 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17846 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17847 uint8_t** ppstop = parm + nptrs; \
17848 if (!start_useful || (uint8_t*)ppstop > (start)) \
17850 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17855 } while (parm < ppstop); \
17857 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17863 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17865 // 1 thing to note about this macro:
17866 // 1) you can use *parm safely but in general you don't want to use parm
17867 // because for the collectible types it's not an address on the managed heap.
17868 #ifndef COLLECTIBLE_CLASS
17869 #define go_through_object_cl(mt,o,size,parm,exp) \
17871 if (header(o)->ContainsPointers()) \
17873 go_through_object_nostart(mt,o,size,parm,exp); \
17876 #else //COLLECTIBLE_CLASS
17877 #define go_through_object_cl(mt,o,size,parm,exp) \
17879 if (header(o)->Collectible()) \
17881 uint8_t* class_obj = get_class_object (o); \
17882 uint8_t** parm = &class_obj; \
17883 do {exp} while (false); \
17885 if (header(o)->ContainsPointers()) \
17887 go_through_object_nostart(mt,o,size,parm,exp); \
17890 #endif //COLLECTIBLE_CLASS
17892 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17893 void gc_heap::enque_pinned_plug (uint8_t* plug,
17894 BOOL save_pre_plug_info_p,
17895 uint8_t* last_object_in_last_plug)
17897 if (mark_stack_array_length <= mark_stack_tos)
17899 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17901 // we don't want to continue here due to security
17902 // risks. This happens very rarely and fixing it in the
17903 // way so that we can continue is a bit involved and will
17904 // not be done in Dev10.
17905 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17909 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17910 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17911 mark& m = mark_stack_array[mark_stack_tos];
17913 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17914 m.saved_pre_p = save_pre_plug_info_p;
17916 if (save_pre_plug_info_p)
17919 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17921 clear_plug_padded (last_object_in_last_plug);
17922 #endif //SHORT_PLUGS
17923 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17926 set_plug_padded (last_object_in_last_plug);
17927 #endif //SHORT_PLUGS
17929 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17931 // If the last object in the last plug is too short, it requires special handling.
17932 size_t last_obj_size = plug - last_object_in_last_plug;
17933 if (last_obj_size < min_pre_pin_obj_size)
17935 record_interesting_data_point (idp_pre_short);
17938 record_interesting_data_point (idp_pre_short_padded);
17939 #endif //SHORT_PLUGS
17940 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17941 last_object_in_last_plug, plug));
17942 // Need to set the short bit regardless of having refs or not because we need to
17943 // indicate that this object is not walkable.
17946 #ifdef COLLECTIBLE_CLASS
17947 if (is_collectible (last_object_in_last_plug))
17949 m.set_pre_short_collectible();
17951 #endif //COLLECTIBLE_CLASS
17953 if (contain_pointers (last_object_in_last_plug))
17955 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17957 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17959 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17960 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17961 m.set_pre_short_bit (gap_offset);
17968 m.saved_post_p = FALSE;
17971 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17973 UNREFERENCED_PARAMETER(last_pinned_plug);
17975 mark& m = mark_stack_array[mark_stack_tos - 1];
17976 assert (last_pinned_plug == m.first);
17977 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17980 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17982 clear_plug_padded (last_object_in_last_plug);
17983 #endif //SHORT_PLUGS
17984 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17987 set_plug_padded (last_object_in_last_plug);
17988 #endif //SHORT_PLUGS
17990 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17992 // This is important - we need to clear all bits here except the last one.
17993 m.saved_post_p = TRUE;
17996 m.saved_post_plug_debug.gap = 1;
17999 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
18001 size_t last_obj_size = post_plug - last_object_in_last_plug;
18002 if (last_obj_size < min_pre_pin_obj_size)
18004 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
18005 record_interesting_data_point (idp_post_short);
18008 record_interesting_data_point (idp_post_short_padded);
18009 #endif //SHORT_PLUGS
18010 m.set_post_short();
18011 #if defined (_DEBUG) && defined (VERIFY_HEAP)
18012 verify_pinned_queue_p = TRUE;
18013 #endif // _DEBUG && VERIFY_HEAP
18015 #ifdef COLLECTIBLE_CLASS
18016 if (is_collectible (last_object_in_last_plug))
18018 m.set_post_short_collectible();
18020 #endif //COLLECTIBLE_CLASS
18022 if (contain_pointers (last_object_in_last_plug))
18024 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
18026 // TODO: since we won't be able to walk this object in relocation, we still need to
18027 // take care of collectible assemblies here.
18028 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
18030 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
18031 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
18032 m.set_post_short_bit (gap_offset);
18041 __declspec(naked) void __fastcall Prefetch(void* addr)
18049 inline void Prefetch (void* addr)
18051 UNREFERENCED_PARAMETER(addr);
18056 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
18058 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
18061 #endif //MH_SC_MARK
18065 #define partial_object 3
18067 uint8_t* ref_from_slot (uint8_t* r)
18069 return (uint8_t*)((size_t)r & ~(stolen | partial));
18072 BOOL stolen_p (uint8_t* r)
18074 return (((size_t)r&2) && !((size_t)r&1));
18077 BOOL ready_p (uint8_t* r)
18079 return ((size_t)r != 1);
18082 BOOL partial_p (uint8_t* r)
18084 return (((size_t)r&1) && !((size_t)r&2));
18087 BOOL straight_ref_p (uint8_t* r)
18089 return (!stolen_p (r) && !partial_p (r));
18092 BOOL partial_object_p (uint8_t* r)
18094 return (((size_t)r & partial_object) == partial_object);
18097 BOOL ref_p (uint8_t* r)
18099 return (straight_ref_p (r) || partial_object_p (r));
18102 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
18104 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
18105 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
18106 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
18107 #ifdef SORT_MARK_STACK
18108 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
18109 #endif //SORT_MARK_STACK
18111 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
18112 // update mark list.
18113 BOOL full_p = (settings.condemned_generation == max_generation);
18115 assert ((start >= oo) && (start < oo+size(oo)));
18118 *mark_stack_tos = oo;
18119 #endif //!MH_SC_MARK
18123 #ifdef MULTIPLE_HEAPS
18124 #else //MULTIPLE_HEAPS
18125 const int thread = 0;
18126 #endif //MULTIPLE_HEAPS
18128 if (oo && ((size_t)oo != 4))
18136 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18138 BOOL overflow_p = FALSE;
18140 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18142 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18143 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
18149 if (overflow_p == FALSE)
18151 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18153 go_through_object_cl (method_table(oo), oo, s, ppslot,
18155 uint8_t* o = *ppslot;
18157 if (gc_mark (o, gc_low, gc_high))
18161 m_boundary_fullgc (o);
18167 size_t obj_size = size (o);
18168 promoted_bytes (thread) += obj_size;
18169 if (contain_pointers_or_collectible (o))
18171 *(mark_stack_tos++) = o;
18179 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18180 min_overflow_address = min (min_overflow_address, oo);
18181 max_overflow_address = max (max_overflow_address, oo);
18186 if (partial_p (oo))
18188 start = ref_from_slot (oo);
18189 oo = ref_from_slot (*(--mark_stack_tos));
18190 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18191 assert ((oo < start) && (start < (oo + size (oo))));
18193 #ifdef COLLECTIBLE_CLASS
18196 // If there's a class object, push it now. We are guaranteed to have the slot since
18197 // we just popped one object off.
18198 if (is_collectible (oo))
18200 uint8_t* class_obj = get_class_object (oo);
18201 if (gc_mark (class_obj, gc_low, gc_high))
18205 m_boundary_fullgc (class_obj);
18209 m_boundary (class_obj);
18212 size_t obj_size = size (class_obj);
18213 promoted_bytes (thread) += obj_size;
18214 *(mark_stack_tos++) = class_obj;
18215 // The code below expects that the oo is still stored in the stack slot that was
18216 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
18217 // But the class_obj has just overwritten that stack slot and so the oo needs to
18218 // be stored to the new slot that's pointed to by the mark_stack_tos.
18219 *mark_stack_tos = oo;
18223 if (!contain_pointers (oo))
18228 #endif //COLLECTIBLE_CLASS
18232 BOOL overflow_p = FALSE;
18234 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18238 if (overflow_p == FALSE)
18240 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18242 //push the object and its current
18243 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
18247 *(place) = (uint8_t*)partial;
18248 #endif //MH_SC_MARK
18249 int i = num_partial_refs;
18250 uint8_t* ref_to_continue = 0;
18252 go_through_object (method_table(oo), oo, s, ppslot,
18253 start, use_start, (oo + s),
18255 uint8_t* o = *ppslot;
18257 if (gc_mark (o, gc_low, gc_high))
18261 m_boundary_fullgc (o);
18267 size_t obj_size = size (o);
18268 promoted_bytes (thread) += obj_size;
18269 if (contain_pointers_or_collectible (o))
18271 *(mark_stack_tos++) = o;
18274 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
18283 //we are finished with this object
18284 assert (ref_to_continue == 0);
18286 assert ((*(place-1)) == (uint8_t*)0);
18289 #endif //MH_SC_MARK
18291 // shouldn't we decrease tos by 2 here??
18294 if (ref_to_continue)
18298 assert ((*(place-1)) == (uint8_t*)0);
18299 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
18300 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
18301 #endif //MH_SC_MARK
18302 *place = ref_to_continue;
18307 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18308 min_overflow_address = min (min_overflow_address, oo);
18309 max_overflow_address = max (max_overflow_address, oo);
18312 #ifdef SORT_MARK_STACK
18313 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18315 rqsort1 (sorted_tos, mark_stack_tos-1);
18316 sorted_tos = mark_stack_tos-1;
18318 #endif //SORT_MARK_STACK
18321 if (!(mark_stack_empty_p()))
18323 oo = *(--mark_stack_tos);
18326 #ifdef SORT_MARK_STACK
18327 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
18328 #endif //SORT_MARK_STACK
18336 BOOL same_numa_node_p (int hn1, int hn2)
18338 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
18341 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
18343 int hn = (current_buddy+1)%n_heaps;
18344 while (hn != current_buddy)
18346 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
18348 hn = (hn+1)%n_heaps;
18350 return current_buddy;
18354 gc_heap::mark_steal()
18356 mark_stack_busy() = 0;
18357 //clear the mark stack in the snooping range
18358 for (int i = 0; i < max_snoop_level; i++)
18360 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18363 //pick the next heap as our buddy
18364 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
18367 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
18368 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18369 #endif //SNOOP_STATS
18371 int idle_loop_count = 0;
18372 int first_not_ready_level = 0;
18376 gc_heap* hp = g_heaps [thpn];
18377 int level = first_not_ready_level;
18378 first_not_ready_level = 0;
18380 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
18382 idle_loop_count = 0;
18384 snoop_stat.busy_count++;
18385 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
18386 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
18387 #endif //SNOOP_STATS
18389 uint8_t* o = ref_mark_stack (hp, level);
18391 uint8_t* start = o;
18394 mark_stack_busy() = 1;
18396 BOOL success = TRUE;
18397 uint8_t* next = (ref_mark_stack (hp, level+1));
18400 if (((size_t)o > 4) && !partial_object_p (o))
18402 //this is a normal object, not a partial mark tuple
18403 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
18404 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
18406 snoop_stat.interlocked_count++;
18408 snoop_stat.normal_count++;
18409 #endif //SNOOP_STATS
18413 //it is a stolen entry, or beginning/ending of a partial mark
18416 snoop_stat.stolen_or_pm_count++;
18417 #endif //SNOOP_STATS
18421 else if (stolen_p (next))
18423 //ignore the stolen guy and go to the next level
18427 snoop_stat.stolen_entry_count++;
18428 #endif //SNOOP_STATS
18432 assert (partial_p (next));
18433 start = ref_from_slot (next);
18434 //re-read the object
18435 o = ref_from_slot (ref_mark_stack (hp, level));
18439 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
18441 snoop_stat.interlocked_count++;
18444 snoop_stat.partial_mark_parent_count++;
18446 #endif //SNOOP_STATS
18450 // stack is not ready, or o is completely different from the last time we read from this stack level.
18451 // go up 2 levels to steal children or totally unrelated objects.
18453 if (first_not_ready_level == 0)
18455 first_not_ready_level = level;
18459 snoop_stat.pm_not_ready_count++;
18460 #endif //SNOOP_STATS
18467 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18468 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18469 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18470 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18471 #endif //SNOOP_STATS
18473 mark_object_simple1 (o, start, heap_number);
18476 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18477 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18478 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18479 #endif //SNOOP_STATS
18481 mark_stack_busy() = 0;
18483 //clear the mark stack in snooping range
18484 for (int i = 0; i < max_snoop_level; i++)
18486 if (((uint8_t**)mark_stack_array)[i] != 0)
18488 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18490 snoop_stat.stack_bottom_clear_count++;
18491 #endif //SNOOP_STATS
18497 mark_stack_busy() = 0;
18501 //slot is either partial or stolen
18505 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18509 if (!hp->mark_stack_busy())
18511 first_not_ready_level = 0;
18514 if ((idle_loop_count % (6) )==1)
18517 snoop_stat.switch_to_thread_count++;
18518 #endif //SNOOP_STATS
18519 GCToOSInterface::Sleep(1);
18521 int free_count = 1;
18523 snoop_stat.stack_idle_count++;
18524 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18525 #endif //SNOOP_STATS
18526 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18528 if (!((g_heaps [hpn])->mark_stack_busy()))
18532 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18533 #endif //SNOOP_STATS
18535 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18540 hpn = (hpn+1)%n_heaps;
18543 if (free_count == n_heaps)
18552 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18555 snoop_stat.check_level_count++;
18556 #endif //SNOOP_STATS
18557 return (next_heap->mark_stack_busy()>=1);
18559 #endif //MH_SC_MARK
18562 void gc_heap::print_snoop_stat()
18564 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18565 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18566 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18567 snoop_stat.heap_index,
18568 snoop_stat.objects_checked_count,
18569 snoop_stat.zero_ref_count,
18570 snoop_stat.objects_marked_count,
18571 snoop_stat.stolen_stack_count,
18572 snoop_stat.partial_stack_count,
18573 snoop_stat.normal_stack_count,
18574 snoop_stat.non_stack_count));
18575 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18576 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18577 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18578 snoop_stat.heap_index,
18579 snoop_stat.check_level_count,
18580 snoop_stat.busy_count,
18581 snoop_stat.interlocked_count,
18582 snoop_stat.partial_mark_parent_count,
18583 snoop_stat.stolen_or_pm_count,
18584 snoop_stat.stolen_entry_count,
18585 snoop_stat.pm_not_ready_count,
18586 snoop_stat.normal_count,
18587 snoop_stat.stack_bottom_clear_count));
18589 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18590 "heap", "check", "zero", "mark", "idle", "switch");
18591 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18592 snoop_stat.heap_index,
18593 snoop_stat.objects_checked_count,
18594 snoop_stat.zero_ref_count,
18595 snoop_stat.objects_marked_count,
18596 snoop_stat.stack_idle_count,
18597 snoop_stat.switch_to_thread_count);
18598 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18599 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18600 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18601 snoop_stat.heap_index,
18602 snoop_stat.check_level_count,
18603 snoop_stat.busy_count,
18604 snoop_stat.interlocked_count,
18605 snoop_stat.partial_mark_parent_count,
18606 snoop_stat.stolen_or_pm_count,
18607 snoop_stat.stolen_entry_count,
18608 snoop_stat.pm_not_ready_count,
18609 snoop_stat.normal_count,
18610 snoop_stat.stack_bottom_clear_count);
18612 #endif //SNOOP_STATS
18614 #ifdef HEAP_ANALYZE
18616 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18618 if (!internal_root_array)
18620 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18621 if (!internal_root_array)
18623 heap_analyze_success = FALSE;
18627 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18629 size_t new_size = 2*internal_root_array_length;
18631 uint64_t available_physical = 0;
18632 get_memory_info (NULL, &available_physical);
18633 if (new_size > (size_t)(available_physical / 10))
18635 heap_analyze_success = FALSE;
18639 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18642 memcpy (tmp, internal_root_array,
18643 internal_root_array_length*sizeof (uint8_t*));
18644 delete[] internal_root_array;
18645 internal_root_array = tmp;
18646 internal_root_array_length = new_size;
18650 heap_analyze_success = FALSE;
18655 if (heap_analyze_success)
18657 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18659 uint8_t* ref = (uint8_t*)po;
18660 if (!current_obj ||
18661 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18663 gc_heap* hp = gc_heap::heap_of (ref);
18664 current_obj = hp->find_object (ref, hp->lowest_address);
18665 current_obj_size = size (current_obj);
18667 internal_root_array[internal_root_array_index] = current_obj;
18668 internal_root_array_index++;
18672 mark_object_simple (po THREAD_NUMBER_ARG);
18674 #endif //HEAP_ANALYZE
18676 //this method assumes that *po is in the [low. high[ range
18678 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18681 #ifdef MULTIPLE_HEAPS
18682 #else //MULTIPLE_HEAPS
18683 const int thread = 0;
18684 #endif //MULTIPLE_HEAPS
18687 snoop_stat.objects_checked_count++;
18688 #endif //SNOOP_STATS
18693 size_t s = size (o);
18694 promoted_bytes (thread) += s;
18696 go_through_object_cl (method_table(o), o, s, poo,
18698 uint8_t* oo = *poo;
18699 if (gc_mark (oo, gc_low, gc_high))
18702 size_t obj_size = size (oo);
18703 promoted_bytes (thread) += obj_size;
18705 if (contain_pointers_or_collectible (oo))
18706 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18716 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18718 if ((o >= gc_low) && (o < gc_high))
18719 mark_object_simple (&o THREAD_NUMBER_ARG);
18720 #ifdef MULTIPLE_HEAPS
18724 gc_heap* hp = heap_of (o);
18726 if ((o >= hp->gc_low) && (o < hp->gc_high))
18727 mark_object_simple (&o THREAD_NUMBER_ARG);
18729 #endif //MULTIPLE_HEAPS
18734 #ifdef BACKGROUND_GC
18736 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18738 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18740 #ifdef SORT_MARK_STACK
18741 uint8_t** sorted_tos = background_mark_stack_array;
18742 #endif //SORT_MARK_STACK
18744 background_mark_stack_tos = background_mark_stack_array;
18748 #ifdef MULTIPLE_HEAPS
18749 #else //MULTIPLE_HEAPS
18750 const int thread = 0;
18751 #endif //MULTIPLE_HEAPS
18755 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18757 BOOL overflow_p = FALSE;
18759 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18761 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18762 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18763 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18765 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18767 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18771 bgc_overflow_count++;
18776 if (overflow_p == FALSE)
18778 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18780 go_through_object_cl (method_table(oo), oo, s, ppslot,
18782 uint8_t* o = *ppslot;
18784 if (background_mark (o,
18785 background_saved_lowest_address,
18786 background_saved_highest_address))
18789 size_t obj_size = size (o);
18790 bpromoted_bytes (thread) += obj_size;
18791 if (contain_pointers_or_collectible (o))
18793 *(background_mark_stack_tos++) = o;
18802 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18803 background_min_overflow_address = min (background_min_overflow_address, oo);
18804 background_max_overflow_address = max (background_max_overflow_address, oo);
18809 uint8_t* start = oo;
18810 if ((size_t)oo & 1)
18812 oo = (uint8_t*)((size_t)oo & ~1);
18813 start = *(--background_mark_stack_tos);
18814 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18816 #ifdef COLLECTIBLE_CLASS
18819 // If there's a class object, push it now. We are guaranteed to have the slot since
18820 // we just popped one object off.
18821 if (is_collectible (oo))
18823 uint8_t* class_obj = get_class_object (oo);
18824 if (background_mark (class_obj,
18825 background_saved_lowest_address,
18826 background_saved_highest_address))
18828 size_t obj_size = size (class_obj);
18829 bpromoted_bytes (thread) += obj_size;
18831 *(background_mark_stack_tos++) = class_obj;
18835 if (!contain_pointers (oo))
18840 #endif //COLLECTIBLE_CLASS
18844 BOOL overflow_p = FALSE;
18846 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18848 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18849 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18851 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18853 (size_t)(mark_stack_limit - background_mark_stack_tos),
18859 bgc_overflow_count++;
18862 if (overflow_p == FALSE)
18864 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18866 //push the object and its current
18867 uint8_t** place = background_mark_stack_tos++;
18869 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18871 int i = num_partial_refs;
18873 go_through_object (method_table(oo), oo, s, ppslot,
18874 start, use_start, (oo + s),
18876 uint8_t* o = *ppslot;
18879 if (background_mark (o,
18880 background_saved_lowest_address,
18881 background_saved_highest_address))
18884 size_t obj_size = size (o);
18885 bpromoted_bytes (thread) += obj_size;
18886 if (contain_pointers_or_collectible (o))
18888 *(background_mark_stack_tos++) = o;
18892 *place = (uint8_t*)(ppslot+1);
18901 //we are finished with this object
18909 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18910 background_min_overflow_address = min (background_min_overflow_address, oo);
18911 background_max_overflow_address = max (background_max_overflow_address, oo);
18915 #ifdef SORT_MARK_STACK
18916 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18918 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18919 sorted_tos = background_mark_stack_tos-1;
18921 #endif //SORT_MARK_STACK
18923 #ifdef COLLECTIBLE_CLASS
18925 #endif // COLLECTIBLE_CLASS
18928 if (!(background_mark_stack_tos == background_mark_stack_array))
18930 oo = *(--background_mark_stack_tos);
18932 #ifdef SORT_MARK_STACK
18933 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18934 #endif //SORT_MARK_STACK
18940 assert (background_mark_stack_tos == background_mark_stack_array);
18945 //this version is different than the foreground GC because
18946 //it can't keep pointers to the inside of an object
18947 //while calling background_mark_simple1. The object could be moved
18948 //by an intervening foreground gc.
18949 //this method assumes that *po is in the [low. high[ range
18951 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18953 #ifdef MULTIPLE_HEAPS
18954 #else //MULTIPLE_HEAPS
18955 const int thread = 0;
18956 #endif //MULTIPLE_HEAPS
18958 dprintf (3, ("bmarking %Ix", o));
18960 if (background_mark1 (o))
18963 size_t s = size (o);
18964 bpromoted_bytes (thread) += s;
18966 if (contain_pointers_or_collectible (o))
18968 background_mark_simple1 (o THREAD_NUMBER_ARG);
18975 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18977 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18979 background_mark_simple (o THREAD_NUMBER_ARG);
18985 dprintf (3, ("or-%Ix", o));
18991 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18993 UNREFERENCED_PARAMETER(sc);
18995 assert (settings.concurrent);
18996 uint8_t* o = (uint8_t*)object;
18998 gc_heap* hp = gc_heap::heap_of (o);
18999 #ifdef INTERIOR_POINTERS
19000 if (flags & GC_CALL_INTERIOR)
19002 o = hp->find_object (o, background_saved_lowest_address);
19004 #endif //INTERIOR_POINTERS
19006 if (!background_object_marked (o, FALSE))
19012 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
19014 UNREFERENCED_PARAMETER(sc);
19015 //in order to save space on the array, mark the object,
19016 //knowing that it will be visited later
19017 assert (settings.concurrent);
19019 THREAD_NUMBER_FROM_CONTEXT;
19020 #ifndef MULTIPLE_HEAPS
19021 const int thread = 0;
19022 #endif //!MULTIPLE_HEAPS
19024 uint8_t* o = (uint8_t*)*ppObject;
19029 #ifdef DEBUG_DestroyedHandleValue
19030 // we can race with destroy handle during concurrent scan
19031 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
19033 #endif //DEBUG_DestroyedHandleValue
19037 gc_heap* hp = gc_heap::heap_of (o);
19039 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
19044 #ifdef INTERIOR_POINTERS
19045 if (flags & GC_CALL_INTERIOR)
19047 o = hp->find_object (o, hp->background_saved_lowest_address);
19051 #endif //INTERIOR_POINTERS
19053 #ifdef FEATURE_CONSERVATIVE_GC
19054 // For conservative GC, a value on stack may point to middle of a free object.
19055 // In this case, we don't need to promote the pointer.
19056 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
19060 #endif //FEATURE_CONSERVATIVE_GC
19063 ((CObjectHeader*)o)->Validate();
19066 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
19068 //needs to be called before the marking because it is possible for a foreground
19069 //gc to take place during the mark and move the object
19070 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
19072 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
19075 //used by the ephemeral collection to scan the local background structures
19076 //containing references.
19078 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
19084 pSC->thread_number = hn;
19086 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19087 pSC->pCurrentDomain = 0;
19090 BOOL relocate_p = (fn == &GCHeap::Relocate);
19092 dprintf (3, ("Scanning background mark list"));
19095 size_t mark_list_finger = 0;
19096 while (mark_list_finger < c_mark_list_index)
19098 uint8_t** o = &c_mark_list [mark_list_finger];
19101 // We may not be able to calculate the size during relocate as POPO
19102 // may have written over the object.
19103 size_t s = size (*o);
19104 assert (Align (s) >= Align (min_obj_size));
19105 dprintf(3,("background root %Ix", (size_t)*o));
19107 (*fn) ((Object**)o, pSC, 0);
19108 mark_list_finger++;
19111 //scan the mark stack
19112 dprintf (3, ("Scanning background mark stack"));
19114 uint8_t** finger = background_mark_stack_array;
19115 while (finger < background_mark_stack_tos)
19117 if ((finger + 1) < background_mark_stack_tos)
19119 // We need to check for the partial mark case here.
19120 uint8_t* parent_obj = *(finger + 1);
19121 if ((size_t)parent_obj & 1)
19123 uint8_t* place = *finger;
19124 size_t place_offset = 0;
19125 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
19129 *(finger + 1) = real_parent_obj;
19130 place_offset = place - real_parent_obj;
19131 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
19132 (*fn) ((Object**)(finger + 1), pSC, 0);
19133 real_parent_obj = *(finger + 1);
19134 *finger = real_parent_obj + place_offset;
19135 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
19136 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
19140 uint8_t** temp = &real_parent_obj;
19141 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
19142 (*fn) ((Object**)temp, pSC, 0);
19149 dprintf(3,("background root %Ix", (size_t)*finger));
19150 (*fn) ((Object**)finger, pSC, 0);
19156 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
19158 if (contain_pointers (oo))
19160 size_t total_refs = 0;
19161 size_t s = size (oo);
19162 go_through_object_nostart (method_table(oo), oo, s, po,
19166 background_mark_object (o THREAD_NUMBER_ARG);
19170 dprintf (3,("Background marking through %Ix went through %Id refs",
19176 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
19178 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
19180 // for now we stop at where gen1 started when we started processing
19181 return background_min_soh_overflow_address;
19185 return heap_segment_allocated (seg);
19189 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
19192 BOOL small_object_p)
19196 if (small_object_p)
19198 if (in_range_for_segment (min_add, seg))
19200 // min_add was the beginning of gen1 when we did the concurrent
19201 // overflow. Now we could be in a situation where min_add is
19202 // actually the same as allocated for that segment (because
19203 // we expanded heap), in which case we can not call
19204 // find first on this address or we will AV.
19205 if (min_add >= heap_segment_allocated (seg))
19211 if (concurrent_p &&
19212 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
19214 return background_min_soh_overflow_address;
19218 o = find_first_object (min_add, heap_segment_mem (seg));
19225 o = max (heap_segment_mem (seg), min_add);
19229 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
19230 uint8_t* min_add, uint8_t* max_add,
19235 current_bgc_state = bgc_overflow_soh;
19238 size_t total_marked_objects = 0;
19240 #ifdef MULTIPLE_HEAPS
19241 int thread = heap_number;
19242 #endif //MULTIPLE_HEAPS
19244 exclusive_sync* loh_alloc_lock = 0;
19246 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19247 #ifdef MULTIPLE_HEAPS
19248 // We don't have each heap scan all heaps concurrently because we are worried about
19249 // multiple threads calling things like find_first_object.
19250 int h_start = (concurrent_p ? heap_number : 0);
19251 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
19252 for (int hi = h_start; hi < h_end; hi++)
19254 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
19260 #endif //MULTIPLE_HEAPS
19261 BOOL small_object_segments = TRUE;
19262 int align_const = get_alignment_constant (small_object_segments);
19263 generation* gen = hp->generation_of (condemned_gen_number);
19264 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19265 PREFIX_ASSUME(seg != NULL);
19266 loh_alloc_lock = hp->bgc_alloc_lock;
19268 uint8_t* o = hp->background_first_overflow (min_add,
19271 small_object_segments);
19275 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
19277 dprintf (3, ("considering %Ix", (size_t)o));
19281 if (concurrent_p && !small_object_segments)
19283 loh_alloc_lock->bgc_mark_set (o);
19285 if (((CObjectHeader*)o)->IsFree())
19287 s = unused_array_size (o);
19299 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
19301 total_marked_objects++;
19302 go_through_object_cl (method_table(o), o, s, poo,
19303 uint8_t* oo = *poo;
19304 background_mark_object (oo THREAD_NUMBER_ARG);
19308 if (concurrent_p && !small_object_segments)
19310 loh_alloc_lock->bgc_mark_done ();
19313 o = o + Align (s, align_const);
19321 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
19322 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
19324 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
19325 (seg = heap_segment_next_in_range (seg)) == 0)
19327 if (small_object_segments)
19331 current_bgc_state = bgc_overflow_loh;
19334 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
19335 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19336 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
19337 total_marked_objects = 0;
19338 small_object_segments = FALSE;
19339 align_const = get_alignment_constant (small_object_segments);
19340 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19342 PREFIX_ASSUME(seg != NULL);
19344 o = max (heap_segment_mem (seg), min_add);
19349 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
19350 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19356 o = hp->background_first_overflow (min_add,
19359 small_object_segments);
19366 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
19368 BOOL grow_mark_array_p = TRUE;
19372 assert (!processed_soh_overflow_p);
19374 if ((background_max_overflow_address != 0) &&
19375 (background_min_overflow_address != MAX_PTR))
19377 // We have overflow to process but we know we can't process the ephemeral generations
19378 // now (we actually could process till the current gen1 start but since we are going to
19379 // make overflow per segment, for now I'll just stop at the saved gen1 start.
19380 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
19381 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
19382 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
19387 assert ((saved_overflow_ephemeral_seg == 0) ||
19388 ((background_max_soh_overflow_address != 0) &&
19389 (background_min_soh_overflow_address != MAX_PTR)));
19391 if (!processed_soh_overflow_p)
19393 // if there was no more overflow we just need to process what we didn't process
19394 // on the saved ephemeral segment.
19395 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
19397 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
19398 grow_mark_array_p = FALSE;
19401 background_min_overflow_address = min (background_min_overflow_address,
19402 background_min_soh_overflow_address);
19403 background_max_overflow_address = max (background_max_overflow_address,
19404 background_max_soh_overflow_address);
19405 processed_soh_overflow_p = TRUE;
19409 BOOL overflow_p = FALSE;
19411 if ((! ((background_max_overflow_address == 0)) ||
19412 ! ((background_min_overflow_address == MAX_PTR))))
19416 if (grow_mark_array_p)
19418 // Try to grow the array.
19419 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
19421 if ((new_size * sizeof(mark)) > 100*1024)
19423 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19425 new_size = min(new_max_size, new_size);
19428 if ((background_mark_stack_array_length < new_size) &&
19429 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
19431 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
19433 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19436 delete background_mark_stack_array;
19437 background_mark_stack_array = tmp;
19438 background_mark_stack_array_length = new_size;
19439 background_mark_stack_tos = background_mark_stack_array;
19445 grow_mark_array_p = TRUE;
19448 uint8_t* min_add = background_min_overflow_address;
19449 uint8_t* max_add = background_max_overflow_address;
19451 background_max_overflow_address = 0;
19452 background_min_overflow_address = MAX_PTR;
19454 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19464 #endif //BACKGROUND_GC
19467 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19469 #ifndef COLLECTIBLE_CLASS
19470 UNREFERENCED_PARAMETER(mark_class_object_p);
19471 BOOL to_mark_class_object = FALSE;
19472 #else //COLLECTIBLE_CLASS
19473 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19474 #endif //COLLECTIBLE_CLASS
19475 if (contain_pointers (oo) || to_mark_class_object)
19477 dprintf(3,( "Marking through %Ix", (size_t)oo));
19478 size_t s = size (oo);
19480 #ifdef COLLECTIBLE_CLASS
19481 if (to_mark_class_object)
19483 uint8_t* class_obj = get_class_object (oo);
19484 mark_object (class_obj THREAD_NUMBER_ARG);
19486 #endif //COLLECTIBLE_CLASS
19488 if (contain_pointers (oo))
19490 go_through_object_nostart (method_table(oo), oo, s, po,
19492 mark_object (o THREAD_NUMBER_ARG);
19498 size_t gc_heap::get_total_heap_size()
19500 size_t total_heap_size = 0;
19502 #ifdef MULTIPLE_HEAPS
19505 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19507 gc_heap* hp2 = gc_heap::g_heaps [hn];
19508 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19511 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19512 #endif //MULTIPLE_HEAPS
19514 return total_heap_size;
19517 size_t gc_heap::get_total_fragmentation()
19519 size_t total_fragmentation = 0;
19521 #ifdef MULTIPLE_HEAPS
19522 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19524 gc_heap* hp = gc_heap::g_heaps[hn];
19525 #else //MULTIPLE_HEAPS
19527 gc_heap* hp = pGenGCHeap;
19528 #endif //MULTIPLE_HEAPS
19529 for (int i = 0; i <= (max_generation + 1); i++)
19531 generation* gen = hp->generation_of (i);
19532 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19536 return total_fragmentation;
19539 size_t gc_heap::get_total_gen_fragmentation (int gen_number)
19541 size_t total_fragmentation = 0;
19543 #ifdef MULTIPLE_HEAPS
19544 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19546 gc_heap* hp = gc_heap::g_heaps[hn];
19547 #else //MULTIPLE_HEAPS
19549 gc_heap* hp = pGenGCHeap;
19550 #endif //MULTIPLE_HEAPS
19551 generation* gen = hp->generation_of (gen_number);
19552 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19555 return total_fragmentation;
19558 size_t gc_heap::get_total_gen_estimated_reclaim (int gen_number)
19560 size_t total_estimated_reclaim = 0;
19562 #ifdef MULTIPLE_HEAPS
19563 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19565 gc_heap* hp = gc_heap::g_heaps[hn];
19566 #else //MULTIPLE_HEAPS
19568 gc_heap* hp = pGenGCHeap;
19569 #endif //MULTIPLE_HEAPS
19570 total_estimated_reclaim += hp->estimated_reclaim (gen_number);
19573 return total_estimated_reclaim;
19576 size_t gc_heap::committed_size()
19578 generation* gen = generation_of (max_generation);
19579 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19580 size_t total_committed = 0;
19584 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19586 seg = heap_segment_next (seg);
19589 if (gen != large_object_generation)
19591 gen = generation_of (max_generation + 1);
19592 seg = generation_start_segment (gen);
19599 return total_committed;
19602 size_t gc_heap::get_total_committed_size()
19604 size_t total_committed = 0;
19606 #ifdef MULTIPLE_HEAPS
19609 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19611 gc_heap* hp = gc_heap::g_heaps [hn];
19612 total_committed += hp->committed_size();
19615 total_committed = committed_size();
19616 #endif //MULTIPLE_HEAPS
19618 return total_committed;
19621 size_t gc_heap::committed_size (bool loh_p, size_t* allocated)
19623 int gen_number = (loh_p ? (max_generation + 1) : max_generation);
19624 generation* gen = generation_of (gen_number);
19625 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19626 size_t total_committed = 0;
19627 size_t total_allocated = 0;
19631 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19632 total_allocated += heap_segment_allocated (seg) - (uint8_t*)seg;
19633 seg = heap_segment_next (seg);
19636 *allocated = total_allocated;
19637 return total_committed;
19640 void gc_heap::get_memory_info (uint32_t* memory_load,
19641 uint64_t* available_physical,
19642 uint64_t* available_page_file)
19644 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19647 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19649 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19650 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19653 //returns TRUE is an overflow happened.
19654 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19656 size_t last_promoted_bytes = promoted_bytes (heap_number);
19657 BOOL overflow_p = FALSE;
19659 if ((! (max_overflow_address == 0) ||
19660 ! (min_overflow_address == MAX_PTR)))
19663 // Try to grow the array.
19665 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19667 if ((new_size * sizeof(mark)) > 100*1024)
19669 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19671 new_size = min(new_max_size, new_size);
19674 if ((mark_stack_array_length < new_size) &&
19675 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19677 mark* tmp = new (nothrow) mark [new_size];
19680 delete mark_stack_array;
19681 mark_stack_array = tmp;
19682 mark_stack_array_length = new_size;
19686 uint8_t* min_add = min_overflow_address;
19687 uint8_t* max_add = max_overflow_address;
19688 max_overflow_address = 0;
19689 min_overflow_address = MAX_PTR;
19690 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19694 size_t current_promoted_bytes = promoted_bytes (heap_number);
19696 if (current_promoted_bytes != last_promoted_bytes)
19697 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19701 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19702 uint8_t* min_add, uint8_t* max_add)
19704 #ifdef MULTIPLE_HEAPS
19705 int thread = heap_number;
19706 #endif //MULTIPLE_HEAPS
19707 BOOL full_p = (condemned_gen_number == max_generation);
19709 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19710 #ifdef MULTIPLE_HEAPS
19711 for (int hi = 0; hi < n_heaps; hi++)
19713 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19719 #endif //MULTIPLE_HEAPS
19720 BOOL small_object_segments = TRUE;
19721 int align_const = get_alignment_constant (small_object_segments);
19722 generation* gen = hp->generation_of (condemned_gen_number);
19723 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19725 PREFIX_ASSUME(seg != NULL);
19726 uint8_t* o = max (heap_segment_mem (seg), min_add);
19729 uint8_t* end = heap_segment_allocated (seg);
19731 while ((o < end) && (o <= max_add))
19733 assert ((min_add <= o) && (max_add >= o));
19734 dprintf (3, ("considering %Ix", (size_t)o));
19737 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19740 o = o + Align (size (o), align_const);
19743 if (( seg = heap_segment_next_in_range (seg)) == 0)
19745 if (small_object_segments && full_p)
19747 small_object_segments = FALSE;
19748 align_const = get_alignment_constant (small_object_segments);
19749 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19751 PREFIX_ASSUME(seg != NULL);
19753 o = max (heap_segment_mem (seg), min_add);
19763 o = max (heap_segment_mem (seg), min_add);
19770 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19771 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19772 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19773 // promotion scan multiple times.
19774 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19775 // also has the effect of processing any mark stack overflow.
19777 #ifdef MULTIPLE_HEAPS
19778 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19779 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19780 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19782 // Define some static variables used for synchronization in the method below. These should really be defined
19783 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19785 // A note about the synchronization used within this method. Communication between the worker threads is
19786 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19787 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19788 // protection of a join.
19789 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19790 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19791 static VOLATILE(BOOL) s_fScanRequired;
19792 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19794 // Whenever we call this method there may have been preceding object promotions. So set
19795 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19796 // based on the how the scanning proceeded).
19797 s_fUnscannedPromotions = TRUE;
19799 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19800 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19801 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19802 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19803 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19804 // as all the others or they'll get out of step).
19807 // The various worker threads are all currently racing in this code. We need to work out if at least
19808 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19809 // dependent handle table when both of the following conditions apply:
19810 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19811 // object happens to correspond to a primary in one of our handles we might potentially have to
19812 // promote the associated secondary).
19813 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19815 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19816 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19817 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19818 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19819 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19820 // the first threads will be racing between reading this value and completing their previous
19821 // iteration's table scan.
19823 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19824 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19825 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19826 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19827 // we're safely joined.
19828 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19829 s_fUnpromotedHandles = TRUE;
19831 // Synchronize all the threads so we can read our state variables safely. The shared variable
19832 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19833 // a single thread inside the join.
19834 gc_t_join.join(this, gc_join_scan_dependent_handles);
19835 if (gc_t_join.joined())
19837 // We're synchronized so it's safe to read our shared state variables. We update another shared
19838 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19839 // the loop. We scan if there has been at least one object promotion since last time and at least
19840 // one thread has a dependent handle table with a potential handle promotion possible.
19841 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19843 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19844 // value for the next call if we're terminating the loop).
19845 s_fUnscannedPromotions = FALSE;
19846 s_fUnpromotedHandles = FALSE;
19848 if (!s_fScanRequired)
19850 // We're terminating the loop. Perform any last operations that require single threaded access.
19851 if (!initial_scan_p)
19853 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19854 // load balance if some of the heaps have an abnormally large workload.
19855 uint8_t* all_heaps_max = 0;
19856 uint8_t* all_heaps_min = MAX_PTR;
19858 for (i = 0; i < n_heaps; i++)
19860 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19861 all_heaps_max = g_heaps[i]->max_overflow_address;
19862 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19863 all_heaps_min = g_heaps[i]->min_overflow_address;
19865 for (i = 0; i < n_heaps; i++)
19867 g_heaps[i]->max_overflow_address = all_heaps_max;
19868 g_heaps[i]->min_overflow_address = all_heaps_min;
19873 // Restart all the workers.
19874 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19875 gc_t_join.restart();
19878 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19879 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19880 // global flag indicating that at least one object promotion may have occurred (the usual comment
19881 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19882 // exit the method since we unconditionally set this variable on method entry anyway).
19883 if (process_mark_overflow(condemned_gen_number))
19884 s_fUnscannedPromotions = TRUE;
19886 // If we decided that no scan was required we can terminate the loop now.
19887 if (!s_fScanRequired)
19890 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19891 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19892 // could miss noting the promotion of some primary objects).
19893 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19894 if (gc_t_join.joined())
19896 // Restart all the workers.
19897 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19898 gc_t_join.restart();
19901 // If the portion of the dependent handle table managed by this worker has handles that could still be
19902 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19903 // could require a rescan of handles on this or other workers.
19904 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19905 if (GCScan::GcDhReScan(sc))
19906 s_fUnscannedPromotions = TRUE;
19909 #else //MULTIPLE_HEAPS
19910 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19911 // threads synchronized.
19912 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19914 UNREFERENCED_PARAMETER(initial_scan_p);
19916 // Whenever we call this method there may have been preceding object promotions. So set
19917 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19918 // based on the how the scanning proceeded).
19919 bool fUnscannedPromotions = true;
19921 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19922 // managed to perform a scan without promoting anything new.
19923 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19925 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19926 fUnscannedPromotions = false;
19928 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19929 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19930 // objects now appear to be promoted and we should set the flag.
19931 if (process_mark_overflow(condemned_gen_number))
19932 fUnscannedPromotions = true;
19934 // Perform the scan and set the flag if any promotions resulted.
19935 if (GCScan::GcDhReScan(sc))
19936 fUnscannedPromotions = true;
19939 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19940 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19942 process_mark_overflow(condemned_gen_number);
19944 #endif //MULTIPLE_HEAPS
19946 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19948 assert (settings.concurrent == FALSE);
19951 sc.thread_number = heap_number;
19952 sc.promotion = TRUE;
19953 sc.concurrent = FALSE;
19955 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19956 BOOL full_p = (condemned_gen_number == max_generation);
19961 start = GetCycleCount32();
19964 int gen_to_init = condemned_gen_number;
19965 if (condemned_gen_number == max_generation)
19967 gen_to_init = max_generation + 1;
19969 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19971 dynamic_data* dd = dynamic_data_of (gen_idx);
19972 dd_begin_data_size (dd) = generation_size (gen_idx) -
19973 dd_fragmentation (dd) -
19974 Align (size (generation_allocation_start (generation_of (gen_idx))));
19975 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19976 dd_survived_size (dd) = 0;
19977 dd_pinned_survived_size (dd) = 0;
19978 dd_artificial_pinned_survived_size (dd) = 0;
19979 dd_added_pinned_size (dd) = 0;
19981 dd_padding_size (dd) = 0;
19982 #endif //SHORT_PLUGS
19983 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19984 dd_num_npinned_plugs (dd) = 0;
19985 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19988 #ifdef FFIND_OBJECT
19989 if (gen0_must_clear_bricks > 0)
19990 gen0_must_clear_bricks--;
19991 #endif //FFIND_OBJECT
19993 size_t last_promoted_bytes = 0;
19995 promoted_bytes (heap_number) = 0;
19996 reset_mark_stack();
19999 memset (&snoop_stat, 0, sizeof(snoop_stat));
20000 snoop_stat.heap_index = heap_number;
20001 #endif //SNOOP_STATS
20006 //initialize the mark stack
20007 for (int i = 0; i < max_snoop_level; i++)
20009 ((uint8_t**)(mark_stack_array))[i] = 0;
20012 mark_stack_busy() = 1;
20014 #endif //MH_SC_MARK
20016 static uint32_t num_sizedrefs = 0;
20019 static BOOL do_mark_steal_p = FALSE;
20020 #endif //MH_SC_MARK
20022 #ifdef MULTIPLE_HEAPS
20023 gc_t_join.join(this, gc_join_begin_mark_phase);
20024 if (gc_t_join.joined())
20026 #endif //MULTIPLE_HEAPS
20028 maxgen_size_inc_p = false;
20030 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
20032 #ifdef MULTIPLE_HEAPS
20037 size_t total_heap_size = get_total_heap_size();
20039 if (total_heap_size > (100 * 1024 * 1024))
20041 do_mark_steal_p = TRUE;
20045 do_mark_steal_p = FALSE;
20050 do_mark_steal_p = FALSE;
20052 #endif //MH_SC_MARK
20054 gc_t_join.restart();
20056 #endif //MULTIPLE_HEAPS
20061 //set up the mark lists from g_mark_list
20062 assert (g_mark_list);
20063 #ifdef MULTIPLE_HEAPS
20064 mark_list = &g_mark_list [heap_number*mark_list_size];
20066 mark_list = g_mark_list;
20067 #endif //MULTIPLE_HEAPS
20068 //dont use the mark list for full gc
20069 //because multiple segments are more complex to handle and the list
20070 //is likely to overflow
20071 if (condemned_gen_number != max_generation)
20072 mark_list_end = &mark_list [mark_list_size-1];
20074 mark_list_end = &mark_list [0];
20075 mark_list_index = &mark_list [0];
20078 #ifndef MULTIPLE_HEAPS
20079 shigh = (uint8_t*) 0;
20081 #endif //MULTIPLE_HEAPS
20083 //%type% category = quote (mark);
20085 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
20087 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20088 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
20089 last_promoted_bytes = promoted_bytes (heap_number);
20091 #ifdef MULTIPLE_HEAPS
20092 gc_t_join.join(this, gc_join_scan_sizedref_done);
20093 if (gc_t_join.joined())
20095 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
20096 gc_t_join.restart();
20098 #endif //MULTIPLE_HEAPS
20101 dprintf(3,("Marking Roots"));
20103 GCScan::GcScanRoots(GCHeap::Promote,
20104 condemned_gen_number, max_generation,
20107 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
20108 last_promoted_bytes = promoted_bytes (heap_number);
20110 #ifdef BACKGROUND_GC
20111 if (recursive_gc_sync::background_running_p())
20113 scan_background_roots (GCHeap::Promote, heap_number, &sc);
20115 #endif //BACKGROUND_GC
20117 #ifdef FEATURE_PREMORTEM_FINALIZATION
20118 dprintf(3, ("Marking finalization data"));
20119 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
20120 #endif // FEATURE_PREMORTEM_FINALIZATION
20122 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
20123 last_promoted_bytes = promoted_bytes (heap_number);
20128 dprintf(3,("Marking handle table"));
20129 GCScan::GcScanHandles(GCHeap::Promote,
20130 condemned_gen_number, max_generation,
20132 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
20133 last_promoted_bytes = promoted_bytes (heap_number);
20137 size_t promoted_before_cards = promoted_bytes (heap_number);
20140 dprintf (3, ("before cards: %Id", promoted_before_cards));
20144 #ifdef MULTIPLE_HEAPS
20145 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
20147 #endif //MULTIPLE_HEAPS
20149 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
20150 // If we are manually managing card bundles, every write to the card table should already be
20151 // accounted for in the card bundle table so there's nothing to update here.
20152 update_card_table_bundle();
20154 if (card_bundles_enabled())
20156 verify_card_bundles();
20159 #ifdef MULTIPLE_HEAPS
20160 gc_t_join.r_restart();
20162 #endif //MULTIPLE_HEAPS
20163 #endif //CARD_BUNDLE
20165 card_fn mark_object_fn = &gc_heap::mark_object_simple;
20166 #ifdef HEAP_ANALYZE
20167 heap_analyze_success = TRUE;
20168 if (heap_analyze_enabled)
20170 internal_root_array_index = 0;
20172 current_obj_size = 0;
20173 mark_object_fn = &gc_heap::ha_mark_object_simple;
20175 #endif //HEAP_ANALYZE
20177 dprintf(3,("Marking cross generation pointers"));
20178 mark_through_cards_for_segments (mark_object_fn, FALSE);
20180 dprintf(3,("Marking cross generation pointers for large objects"));
20181 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
20183 dprintf (3, ("marked by cards: %Id",
20184 (promoted_bytes (heap_number) - promoted_before_cards)));
20185 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
20186 last_promoted_bytes = promoted_bytes (heap_number);
20191 if (do_mark_steal_p)
20195 #endif //MH_SC_MARK
20197 // Dependent handles need to be scanned with a special algorithm (see the header comment on
20198 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
20199 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
20200 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
20201 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
20202 // iterations if required and will also perform processing of any mark stack overflow once the dependent
20203 // handle table has been fully promoted.
20204 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20205 scan_dependent_handles(condemned_gen_number, &sc, true);
20207 #ifdef MULTIPLE_HEAPS
20208 dprintf(3, ("Joining for short weak handle scan"));
20209 gc_t_join.join(this, gc_join_null_dead_short_weak);
20210 if (gc_t_join.joined())
20211 #endif //MULTIPLE_HEAPS
20213 #ifdef HEAP_ANALYZE
20214 heap_analyze_enabled = FALSE;
20215 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
20216 #endif // HEAP_ANALYZE
20217 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
20219 #ifdef MULTIPLE_HEAPS
20222 // we used r_join and need to reinitialize states for it here.
20223 gc_t_join.r_init();
20226 //start all threads on the roots.
20227 dprintf(3, ("Starting all gc thread for short weak handle scan"));
20228 gc_t_join.restart();
20229 #endif //MULTIPLE_HEAPS
20233 // null out the target of short weakref that were not promoted.
20234 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
20236 // MTHTS: keep by single thread
20237 #ifdef MULTIPLE_HEAPS
20238 dprintf(3, ("Joining for finalization"));
20239 gc_t_join.join(this, gc_join_scan_finalization);
20240 if (gc_t_join.joined())
20241 #endif //MULTIPLE_HEAPS
20244 #ifdef MULTIPLE_HEAPS
20245 //start all threads on the roots.
20246 dprintf(3, ("Starting all gc thread for Finalization"));
20247 gc_t_join.restart();
20248 #endif //MULTIPLE_HEAPS
20251 //Handle finalization.
20252 size_t promoted_bytes_live = promoted_bytes (heap_number);
20254 #ifdef FEATURE_PREMORTEM_FINALIZATION
20255 dprintf (3, ("Finalize marking"));
20256 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
20258 GCToEEInterface::DiagWalkFReachableObjects(__this);
20259 #endif // FEATURE_PREMORTEM_FINALIZATION
20261 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
20262 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
20263 scan_dependent_handles(condemned_gen_number, &sc, false);
20265 #ifdef MULTIPLE_HEAPS
20266 dprintf(3, ("Joining for weak pointer deletion"));
20267 gc_t_join.join(this, gc_join_null_dead_long_weak);
20268 if (gc_t_join.joined())
20270 //start all threads on the roots.
20271 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
20272 gc_t_join.restart();
20274 #endif //MULTIPLE_HEAPS
20276 // null out the target of long weakref that were not promoted.
20277 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20279 // MTHTS: keep by single thread
20280 #ifdef MULTIPLE_HEAPS
20282 #ifdef PARALLEL_MARK_LIST_SORT
20283 // unsigned long start = GetCycleCount32();
20285 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
20286 #endif //PARALLEL_MARK_LIST_SORT
20289 dprintf (3, ("Joining for sync block cache entry scanning"));
20290 gc_t_join.join(this, gc_join_null_dead_syncblk);
20291 if (gc_t_join.joined())
20292 #endif //MULTIPLE_HEAPS
20294 // scan for deleted entries in the syncblk cache
20295 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
20297 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
20298 if (g_fEnableAppDomainMonitoring)
20300 size_t promoted_all_heaps = 0;
20301 #ifdef MULTIPLE_HEAPS
20302 for (int i = 0; i < n_heaps; i++)
20304 promoted_all_heaps += promoted_bytes (i);
20307 promoted_all_heaps = promoted_bytes (heap_number);
20308 #endif //MULTIPLE_HEAPS
20309 GCToEEInterface::RecordTotalSurvivedBytes(promoted_all_heaps);
20311 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
20313 #ifdef MULTIPLE_HEAPS
20316 #ifndef PARALLEL_MARK_LIST_SORT
20317 //compact g_mark_list and sort it.
20318 combine_mark_lists();
20319 #endif //PARALLEL_MARK_LIST_SORT
20322 //decide on promotion
20323 if (!settings.promotion)
20326 for (int n = 0; n <= condemned_gen_number;n++)
20328 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
20331 for (int i = 0; i < n_heaps; i++)
20333 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
20335 size_t older_gen_size = (dd_current_size (dd) +
20336 (dd_desired_allocation (dd) -
20337 dd_new_allocation (dd)));
20339 if ((m > (older_gen_size)) ||
20340 (promoted_bytes (i) > m))
20342 settings.promotion = TRUE;
20348 if (do_mark_steal_p)
20350 size_t objects_checked_count = 0;
20351 size_t zero_ref_count = 0;
20352 size_t objects_marked_count = 0;
20353 size_t check_level_count = 0;
20354 size_t busy_count = 0;
20355 size_t interlocked_count = 0;
20356 size_t partial_mark_parent_count = 0;
20357 size_t stolen_or_pm_count = 0;
20358 size_t stolen_entry_count = 0;
20359 size_t pm_not_ready_count = 0;
20360 size_t normal_count = 0;
20361 size_t stack_bottom_clear_count = 0;
20363 for (int i = 0; i < n_heaps; i++)
20365 gc_heap* hp = g_heaps[i];
20366 hp->print_snoop_stat();
20367 objects_checked_count += hp->snoop_stat.objects_checked_count;
20368 zero_ref_count += hp->snoop_stat.zero_ref_count;
20369 objects_marked_count += hp->snoop_stat.objects_marked_count;
20370 check_level_count += hp->snoop_stat.check_level_count;
20371 busy_count += hp->snoop_stat.busy_count;
20372 interlocked_count += hp->snoop_stat.interlocked_count;
20373 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
20374 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
20375 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
20376 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
20377 normal_count += hp->snoop_stat.normal_count;
20378 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
20383 printf ("-------total stats-------\n");
20384 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
20385 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
20386 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
20387 objects_checked_count,
20389 objects_marked_count,
20393 partial_mark_parent_count,
20394 stolen_or_pm_count,
20395 stolen_entry_count,
20396 pm_not_ready_count,
20398 stack_bottom_clear_count);
20400 #endif //SNOOP_STATS
20402 //start all threads.
20403 dprintf(3, ("Starting all threads for end of mark phase"));
20404 gc_t_join.restart();
20405 #else //MULTIPLE_HEAPS
20407 //decide on promotion
20408 if (!settings.promotion)
20411 for (int n = 0; n <= condemned_gen_number;n++)
20413 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
20415 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
20417 size_t older_gen_size = (dd_current_size (dd) +
20418 (dd_desired_allocation (dd) -
20419 dd_new_allocation (dd)));
20421 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
20422 m, promoted_bytes (heap_number), older_gen_size));
20424 if ((m > older_gen_size) ||
20425 (promoted_bytes (heap_number) > m))
20427 settings.promotion = TRUE;
20431 #endif //MULTIPLE_HEAPS
20434 #ifdef MULTIPLE_HEAPS
20436 #ifdef PARALLEL_MARK_LIST_SORT
20437 // start = GetCycleCount32();
20438 merge_mark_lists();
20439 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
20440 #endif //PARALLEL_MARK_LIST_SORT
20442 #endif //MULTIPLE_HEAPS
20444 #ifdef BACKGROUND_GC
20445 total_promoted_bytes = promoted_bytes (heap_number);
20446 #endif //BACKGROUND_GC
20448 promoted_bytes (heap_number) -= promoted_bytes_live;
20451 finish = GetCycleCount32();
20452 mark_time = finish - start;
20455 dprintf(2,("---- End of mark phase ----"));
20459 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
20461 dprintf (3, ("Pinning %Ix", (size_t)o));
20462 if ((o >= low) && (o < high))
20464 dprintf(3,("^%Ix^", (size_t)o));
20467 #ifdef FEATURE_EVENT_TRACE
20468 if(EVENT_ENABLED(PinObjectAtGCTime))
20470 fire_etw_pin_object_event(o, ppObject);
20472 #endif // FEATURE_EVENT_TRACE
20474 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20475 num_pinned_objects++;
20476 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20480 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20481 size_t gc_heap::get_total_pinned_objects()
20483 #ifdef MULTIPLE_HEAPS
20484 size_t total_num_pinned_objects = 0;
20485 for (int i = 0; i < gc_heap::n_heaps; i++)
20487 gc_heap* hp = gc_heap::g_heaps[i];
20488 total_num_pinned_objects += hp->num_pinned_objects;
20490 return total_num_pinned_objects;
20491 #else //MULTIPLE_HEAPS
20492 return num_pinned_objects;
20493 #endif //MULTIPLE_HEAPS
20495 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20497 void gc_heap::reset_mark_stack ()
20499 reset_pinned_queue();
20500 max_overflow_address = 0;
20501 min_overflow_address = MAX_PTR;
20504 #ifdef FEATURE_STRUCTALIGN
20506 // The word with left child, right child, and align info is laid out as follows:
20508 // | upper short word | lower short word |
20509 // |<------------> <----->|<------------> <----->|
20510 // | left child info hi| right child info lo|
20511 // x86: | 10 bits 6 bits| 10 bits 6 bits|
20513 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
20515 // The "align info" encodes two numbers: the required alignment (a power of two)
20516 // and the misalignment (the number of machine words the destination address needs
20517 // to be adjusted by to provide alignment - so this number is always smaller than
20518 // the required alignment). Thus, the two can be represented as the "logical or"
20519 // of the two numbers. Note that the actual pad is computed from the misalignment
20520 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20523 // The number of bits in a brick.
20524 #if defined (_TARGET_AMD64_)
20525 #define brick_bits (12)
20527 #define brick_bits (11)
20528 #endif //_TARGET_AMD64_
20529 C_ASSERT(brick_size == (1 << brick_bits));
20531 // The number of bits needed to represent the offset to a child node.
20532 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20533 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20535 // The number of bits in each of the pad hi, pad lo fields.
20536 #define pad_bits (sizeof(short) * 8 - child_bits)
20538 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20539 #define pad_mask ((1 << pad_bits) - 1)
20540 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20541 #else // FEATURE_STRUCTALIGN
20542 #define child_from_short(w) (w)
20543 #endif // FEATURE_STRUCTALIGN
20546 short node_left_child(uint8_t* node)
20548 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20552 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20554 assert (val > -(ptrdiff_t)brick_size);
20555 assert (val < (ptrdiff_t)brick_size);
20556 assert (Aligned (val));
20557 #ifdef FEATURE_STRUCTALIGN
20558 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20559 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20560 #else // FEATURE_STRUCTALIGN
20561 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20562 #endif // FEATURE_STRUCTALIGN
20563 assert (node_left_child (node) == val);
20567 short node_right_child(uint8_t* node)
20569 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20573 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20575 assert (val > -(ptrdiff_t)brick_size);
20576 assert (val < (ptrdiff_t)brick_size);
20577 assert (Aligned (val));
20578 #ifdef FEATURE_STRUCTALIGN
20579 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20580 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20581 #else // FEATURE_STRUCTALIGN
20582 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20583 #endif // FEATURE_STRUCTALIGN
20584 assert (node_right_child (node) == val);
20587 #ifdef FEATURE_STRUCTALIGN
20588 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20590 // Extract the single-number aligninfo from the fields.
20591 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20592 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20593 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20594 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20596 // Replicate the topmost bit into all lower bits.
20597 ptrdiff_t x = aligninfo;
20603 // Clear all bits but the highest.
20604 requiredAlignment = (int)(x ^ (x >> 1));
20605 pad = aligninfo - requiredAlignment;
20606 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20610 ptrdiff_t node_alignpad (uint8_t* node)
20612 int requiredAlignment;
20613 ptrdiff_t alignpad;
20614 node_aligninfo (node, requiredAlignment, alignpad);
20618 void clear_node_aligninfo (uint8_t* node)
20620 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20621 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20624 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20626 // Encode the alignment requirement and alignment offset as a single number
20627 // as described above.
20628 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20629 assert (Aligned (aligninfo));
20630 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20631 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20633 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20634 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20635 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20637 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20638 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20639 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20642 int requiredAlignment2;
20644 node_aligninfo (node, requiredAlignment2, pad2);
20645 assert (requiredAlignment == requiredAlignment2);
20646 assert (pad == pad2);
20649 #endif // FEATURE_STRUCTALIGN
20652 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20654 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20659 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20661 return (((loh_obj_and_pad*)node)[-1].reloc);
20665 ptrdiff_t node_relocation_distance (uint8_t* node)
20667 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20671 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20673 assert (val == (val & ~3));
20674 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20675 //clear the left bit and the relocation field
20681 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20683 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20685 #ifndef FEATURE_STRUCTALIGN
20686 void set_node_realigned(uint8_t* node)
20688 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20691 void clear_node_realigned(uint8_t* node)
20693 #ifdef RESPECT_LARGE_ALIGNMENT
20694 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20696 UNREFERENCED_PARAMETER(node);
20697 #endif //RESPECT_LARGE_ALIGNMENT
20699 #endif // FEATURE_STRUCTALIGN
20702 size_t node_gap_size (uint8_t* node)
20704 return ((plug_and_gap *)node)[-1].gap;
20707 void set_gap_size (uint8_t* node, size_t size)
20709 assert (Aligned (size));
20711 // clear the 2 uint32_t used by the node.
20712 ((plug_and_gap *)node)[-1].reloc = 0;
20713 ((plug_and_gap *)node)[-1].lr =0;
20714 ((plug_and_gap *)node)[-1].gap = size;
20716 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20720 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20721 uint8_t* tree, uint8_t* last_node)
20723 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20724 (size_t)new_node, brick_of(new_node),
20725 (size_t)tree, brick_of(tree),
20726 (size_t)last_node, brick_of(last_node),
20728 if (power_of_two_p (sequence_number))
20730 set_node_left_child (new_node, (tree - new_node));
20731 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20736 if (oddp (sequence_number))
20738 set_node_right_child (last_node, (new_node - last_node));
20739 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20743 uint8_t* earlier_node = tree;
20744 size_t imax = logcount(sequence_number) - 2;
20745 for (size_t i = 0; i != imax; i++)
20747 earlier_node = earlier_node + node_right_child (earlier_node);
20749 int tmp_offset = node_right_child (earlier_node);
20750 assert (tmp_offset); // should never be empty
20751 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20752 set_node_right_child (earlier_node, (new_node - earlier_node));
20754 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20755 new_node, ((earlier_node + tmp_offset ) - new_node),
20756 earlier_node, (new_node - earlier_node)));
20762 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20763 uint8_t* x, uint8_t* plug_end)
20765 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20766 tree, current_brick, x, plug_end));
20770 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20771 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20772 set_brick (current_brick, (tree - brick_address (current_brick)));
20776 dprintf (3, ("b- %Ix->-1", current_brick));
20777 set_brick (current_brick, -1);
20779 size_t b = 1 + current_brick;
20780 ptrdiff_t offset = 0;
20781 size_t last_br = brick_of (plug_end-1);
20782 current_brick = brick_of (x-1);
20783 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20784 while (b <= current_brick)
20788 set_brick (b, --offset);
20796 return brick_of (x);
20799 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20802 // We should never demote big plugs to gen0.
20803 if (gen == youngest_generation)
20805 heap_segment* seg = ephemeral_heap_segment;
20806 size_t mark_stack_large_bos = mark_stack_bos;
20807 size_t large_plug_pos = 0;
20808 while (mark_stack_large_bos < mark_stack_tos)
20810 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20812 while (mark_stack_bos <= mark_stack_large_bos)
20814 size_t entry = deque_pinned_plug();
20815 size_t len = pinned_len (pinned_plug_of (entry));
20816 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20817 if (len > demotion_plug_len_th)
20819 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20821 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20822 assert(mark_stack_array[entry].len == 0 ||
20823 mark_stack_array[entry].len >= Align(min_obj_size));
20824 generation_allocation_pointer (consing_gen) = plug + len;
20825 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20826 set_allocator_next_pin (consing_gen);
20830 mark_stack_large_bos++;
20835 generation_plan_allocation_start (gen) =
20836 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20837 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20838 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20839 if (next_plug_to_allocate)
20841 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20842 if (allocation_left > dist_to_next_plug)
20844 allocation_left = dist_to_next_plug;
20847 if (allocation_left < Align (min_obj_size))
20849 generation_plan_allocation_start_size (gen) += allocation_left;
20850 generation_allocation_pointer (consing_gen) += allocation_left;
20853 dprintf (2, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20854 generation_plan_allocation_start (gen),
20855 generation_plan_allocation_start_size (gen),
20856 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20857 next_plug_to_allocate));
20860 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20862 BOOL adjacentp = FALSE;
20864 generation_plan_allocation_start (gen) =
20865 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20868 #endif //SHORT_PLUGS
20869 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20871 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20872 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20873 if ((allocation_left < Align (min_obj_size)) &&
20874 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20876 generation_plan_allocation_start_size (gen) += allocation_left;
20877 generation_allocation_pointer (consing_gen) += allocation_left;
20880 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20881 generation_plan_allocation_start (consing_gen),
20882 generation_allocation_pointer (consing_gen),
20883 generation_allocation_limit (consing_gen)));
20886 void gc_heap::plan_generation_starts (generation*& consing_gen)
20888 //make sure that every generation has a planned allocation start
20889 int gen_number = settings.condemned_generation;
20890 while (gen_number >= 0)
20892 if (gen_number < max_generation)
20894 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20896 generation* gen = generation_of (gen_number);
20897 if (0 == generation_plan_allocation_start (gen))
20899 plan_generation_start (gen, consing_gen, 0);
20900 assert (generation_plan_allocation_start (gen));
20904 // now we know the planned allocation size
20905 heap_segment_plan_allocated (ephemeral_heap_segment) =
20906 generation_allocation_pointer (consing_gen);
20909 void gc_heap::advance_pins_for_demotion (generation* gen)
20911 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20912 heap_segment* seg = ephemeral_heap_segment;
20914 if ((!(pinned_plug_que_empty_p())))
20916 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20917 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20918 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20919 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20920 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20921 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20923 while (!pinned_plug_que_empty_p() &&
20924 (pinned_plug (oldest_pin()) < original_youngest_start))
20926 size_t entry = deque_pinned_plug();
20927 size_t len = pinned_len (pinned_plug_of (entry));
20928 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20929 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20930 assert(mark_stack_array[entry].len == 0 ||
20931 mark_stack_array[entry].len >= Align(min_obj_size));
20932 generation_allocation_pointer (gen) = plug + len;
20933 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20934 set_allocator_next_pin (gen);
20936 //Add the size of the pinned plug to the right pinned allocations
20937 //find out which gen this pinned plug came from
20938 int frgn = object_gennum (plug);
20939 if ((frgn != (int)max_generation) && settings.promotion)
20941 int togn = object_gennum_plan (plug);
20942 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20945 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20949 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20950 pinned_len (pinned_plug_of (entry)), plug, len));
20953 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20954 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20958 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20959 int& active_new_gen_number,
20960 int& active_old_gen_number,
20961 generation*& consing_gen,
20962 BOOL& allocate_in_condemned)
20965 if ((active_old_gen_number > 0) &&
20966 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20968 dprintf (2, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20970 if (!pinned_plug_que_empty_p())
20972 dprintf (2, ("oldest pin: %Ix(%Id)",
20973 pinned_plug (oldest_pin()),
20974 (x - pinned_plug (oldest_pin()))));
20977 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20979 active_new_gen_number--;
20982 active_old_gen_number--;
20983 assert ((!settings.promotion) || (active_new_gen_number>0));
20985 if (active_new_gen_number == (max_generation - 1))
20987 #ifdef FREE_USAGE_STATS
20988 if (settings.condemned_generation == max_generation)
20990 // We need to do this before we skip the rest of the pinned plugs.
20991 generation* gen_2 = generation_of (max_generation);
20992 generation* gen_1 = generation_of (max_generation - 1);
20994 size_t total_num_pinned_free_spaces_left = 0;
20996 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20997 for (int j = 0; j < NUM_GEN_POWER2; j++)
20999 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
21003 gen_2->gen_current_pinned_free_spaces[j],
21004 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
21005 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
21007 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
21010 float pinned_free_list_efficiency = 0;
21011 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
21012 if (total_pinned_free_space != 0)
21014 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
21017 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
21019 generation_allocated_in_pinned_free (gen_2),
21020 total_pinned_free_space,
21021 (int)(pinned_free_list_efficiency * 100),
21022 generation_pinned_free_obj_space (gen_2),
21023 total_num_pinned_free_spaces_left));
21025 #endif //FREE_USAGE_STATS
21027 //Go past all of the pinned plugs for this generation.
21028 while (!pinned_plug_que_empty_p() &&
21029 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
21031 size_t entry = deque_pinned_plug();
21032 mark* m = pinned_plug_of (entry);
21033 uint8_t* plug = pinned_plug (m);
21034 size_t len = pinned_len (m);
21035 // detect pinned block in different segment (later) than
21036 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
21037 // adjust the allocation segment along the way (at the end it will
21038 // be the ephemeral segment.
21039 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
21041 PREFIX_ASSUME(nseg != NULL);
21043 while (!((plug >= generation_allocation_pointer (consing_gen))&&
21044 (plug < heap_segment_allocated (nseg))))
21046 //adjust the end of the segment to be the end of the plug
21047 assert (generation_allocation_pointer (consing_gen)>=
21048 heap_segment_mem (nseg));
21049 assert (generation_allocation_pointer (consing_gen)<=
21050 heap_segment_committed (nseg));
21052 heap_segment_plan_allocated (nseg) =
21053 generation_allocation_pointer (consing_gen);
21054 //switch allocation segment
21055 nseg = heap_segment_next_rw (nseg);
21056 generation_allocation_segment (consing_gen) = nseg;
21057 //reset the allocation pointer and limits
21058 generation_allocation_pointer (consing_gen) =
21059 heap_segment_mem (nseg);
21061 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
21062 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
21063 generation_allocation_pointer (consing_gen) = plug + len;
21064 generation_allocation_limit (consing_gen) =
21065 generation_allocation_pointer (consing_gen);
21067 allocate_in_condemned = TRUE;
21068 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
21071 if (active_new_gen_number != max_generation)
21073 if (active_new_gen_number == (max_generation - 1))
21075 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
21076 if (!demote_gen1_p)
21077 advance_pins_for_demotion (consing_gen);
21080 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
21082 dprintf (2, ("process eph: allocated gen%d start at %Ix",
21083 active_new_gen_number,
21084 generation_plan_allocation_start (generation_of (active_new_gen_number))));
21086 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
21088 uint8_t* pplug = pinned_plug (oldest_pin());
21089 if (object_gennum (pplug) > 0)
21091 demotion_low = pplug;
21092 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
21096 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
21104 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
21106 uint8_t* o = heap_segment_mem (seg);
21107 while (o < heap_segment_allocated (seg))
21113 o = o + Align (size (o));
21117 #ifdef FEATURE_BASICFREEZE
21118 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
21120 //go through all of the segment in range and reset the mark bit
21121 //TODO works only on small object segments
21123 heap_segment* seg = start_seg;
21127 if (heap_segment_read_only_p (seg) &&
21128 heap_segment_in_range_p (seg))
21130 #ifdef BACKGROUND_GC
21131 if (settings.concurrent)
21133 seg_clear_mark_array_bits_soh (seg);
21137 seg_clear_mark_bits (seg);
21139 #else //BACKGROUND_GC
21142 if(gc_can_use_concurrent)
21144 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
21145 min (heap_segment_allocated (seg), highest_address),
21146 FALSE); // read_only segments need the mark clear
21149 seg_clear_mark_bits (seg);
21150 #endif //MARK_ARRAY
21152 #endif //BACKGROUND_GC
21154 seg = heap_segment_next (seg);
21157 #endif // FEATURE_BASICFREEZE
21159 #ifdef FEATURE_LOH_COMPACTION
21161 BOOL gc_heap::loh_pinned_plug_que_empty_p()
21163 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
21166 void gc_heap::loh_set_allocator_next_pin()
21168 if (!(loh_pinned_plug_que_empty_p()))
21170 mark* oldest_entry = loh_oldest_pin();
21171 uint8_t* plug = pinned_plug (oldest_entry);
21172 generation* gen = large_object_generation;
21173 if ((plug >= generation_allocation_pointer (gen)) &&
21174 (plug < generation_allocation_limit (gen)))
21176 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
21179 assert (!((plug < generation_allocation_pointer (gen)) &&
21180 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
21184 size_t gc_heap::loh_deque_pinned_plug ()
21186 size_t m = loh_pinned_queue_bos;
21187 loh_pinned_queue_bos++;
21192 mark* gc_heap::loh_pinned_plug_of (size_t bos)
21194 return &loh_pinned_queue[bos];
21198 mark* gc_heap::loh_oldest_pin()
21200 return loh_pinned_plug_of (loh_pinned_queue_bos);
21203 // If we can't grow the queue, then don't compact.
21204 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
21206 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
21208 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
21210 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
21215 dprintf (3, (" P: %Ix(%Id)", plug, len));
21216 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
21219 loh_pinned_queue_tos++;
21220 loh_set_allocator_next_pin();
21225 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
21227 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
21229 (2* AlignQword (loh_padding_obj_size) + size),
21232 (alloc_limit - alloc_pointer)));
21234 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
21237 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
21239 UNREFERENCED_PARAMETER(old_loc);
21241 generation* gen = large_object_generation;
21242 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
21243 generation_allocation_pointer (gen),
21244 generation_allocation_limit (gen),
21249 heap_segment* seg = generation_allocation_segment (gen);
21250 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
21252 if ((!(loh_pinned_plug_que_empty_p()) &&
21253 (generation_allocation_limit (gen) ==
21254 pinned_plug (loh_oldest_pin()))))
21256 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21257 size_t len = pinned_len (m);
21258 uint8_t* plug = pinned_plug (m);
21259 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21260 pinned_len (m) = plug - generation_allocation_pointer (gen);
21261 generation_allocation_pointer (gen) = plug + len;
21263 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21264 loh_set_allocator_next_pin();
21265 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
21266 generation_allocation_pointer (gen),
21267 generation_allocation_limit (gen),
21268 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21273 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
21275 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21276 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
21280 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
21282 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21283 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21284 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
21288 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
21289 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
21291 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
21292 (generation_allocation_pointer (gen) + size)));
21294 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21295 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21297 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
21298 generation_allocation_pointer (gen),
21299 generation_allocation_limit (gen),
21300 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21304 heap_segment* next_seg = heap_segment_next (seg);
21305 assert (generation_allocation_pointer (gen)>=
21306 heap_segment_mem (seg));
21307 // Verify that all pinned plugs for this segment are consumed
21308 if (!loh_pinned_plug_que_empty_p() &&
21309 ((pinned_plug (loh_oldest_pin()) <
21310 heap_segment_allocated (seg)) &&
21311 (pinned_plug (loh_oldest_pin()) >=
21312 generation_allocation_pointer (gen))))
21314 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
21315 pinned_plug (loh_oldest_pin())));
21316 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
21319 assert (generation_allocation_pointer (gen)>=
21320 heap_segment_mem (seg));
21321 assert (generation_allocation_pointer (gen)<=
21322 heap_segment_committed (seg));
21323 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
21327 // for LOH do we want to try starting from the first LOH every time though?
21328 generation_allocation_segment (gen) = next_seg;
21329 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
21330 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21332 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
21333 generation_allocation_pointer (gen),
21334 generation_allocation_limit (gen),
21335 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21339 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
21345 loh_set_allocator_next_pin();
21347 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
21348 generation_allocation_pointer (gen),
21349 generation_allocation_limit (gen),
21350 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21357 assert (generation_allocation_pointer (gen)>=
21358 heap_segment_mem (generation_allocation_segment (gen)));
21359 uint8_t* result = generation_allocation_pointer (gen);
21360 size_t loh_pad = AlignQword (loh_padding_obj_size);
21362 generation_allocation_pointer (gen) += size + loh_pad;
21363 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
21365 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
21366 generation_allocation_pointer (gen),
21367 generation_allocation_limit (gen),
21368 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21370 assert (result + loh_pad);
21371 return result + loh_pad;
21375 BOOL gc_heap::should_compact_loh()
21377 // If hard limit is specified GC will automatically decide if LOH needs to be compacted.
21378 return (heap_hard_limit || loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
21382 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
21384 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
21386 if (all_heaps_compacted_p)
21388 // If the compaction mode says to compact once and we are going to compact LOH,
21389 // we need to revert it back to no compaction.
21390 loh_compaction_mode = loh_compaction_default;
21395 BOOL gc_heap::plan_loh()
21397 if (!loh_pinned_queue)
21399 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
21400 if (!loh_pinned_queue)
21402 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
21403 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
21407 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
21410 if (heap_number == 0)
21411 loh_pinned_queue_decay = LOH_PIN_DECAY;
21413 loh_pinned_queue_tos = 0;
21414 loh_pinned_queue_bos = 0;
21416 generation* gen = large_object_generation;
21417 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21418 PREFIX_ASSUME(start_seg != NULL);
21419 heap_segment* seg = start_seg;
21420 uint8_t* o = generation_allocation_start (gen);
21422 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
21423 generation_size (max_generation + 1),
21424 generation_free_list_space (gen),
21425 generation_free_obj_space (gen)));
21429 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
21430 seg = heap_segment_next (seg);
21435 //Skip the generation gap object
21436 o = o + AlignQword (size (o));
21437 // We don't need to ever realloc gen3 start so don't touch it.
21438 heap_segment_plan_allocated (seg) = o;
21439 generation_allocation_pointer (gen) = o;
21440 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21441 generation_allocation_segment (gen) = start_seg;
21443 uint8_t* free_space_start = o;
21444 uint8_t* free_space_end = o;
21445 uint8_t* new_address = 0;
21449 if (o >= heap_segment_allocated (seg))
21451 seg = heap_segment_next (seg);
21457 o = heap_segment_mem (seg);
21462 free_space_end = o;
21463 size_t size = AlignQword (size (o));
21464 dprintf (1235, ("%Ix(%Id) M", o, size));
21468 // We don't clear the pinned bit yet so we can check in
21469 // compact phase how big a free object we should allocate
21470 // in front of the pinned object. We use the reloc address
21471 // field to store this.
21472 if (!loh_enque_pinned_plug (o, size))
21480 new_address = loh_allocate_in_condemned (o, size);
21483 loh_set_node_relocation_distance (o, (new_address - o));
21484 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
21487 free_space_start = o;
21488 if (o < heap_segment_allocated (seg))
21490 assert (!marked (o));
21495 while (o < heap_segment_allocated (seg) && !marked (o))
21497 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
21498 o = o + AlignQword (size (o));
21503 while (!loh_pinned_plug_que_empty_p())
21505 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21506 size_t len = pinned_len (m);
21507 uint8_t* plug = pinned_plug (m);
21509 // detect pinned block in different segment (later) than
21510 // allocation segment
21511 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
21513 while ((plug < generation_allocation_pointer (gen)) ||
21514 (plug >= heap_segment_allocated (nseg)))
21516 assert ((plug < heap_segment_mem (nseg)) ||
21517 (plug > heap_segment_reserved (nseg)));
21518 //adjust the end of the segment to be the end of the plug
21519 assert (generation_allocation_pointer (gen)>=
21520 heap_segment_mem (nseg));
21521 assert (generation_allocation_pointer (gen)<=
21522 heap_segment_committed (nseg));
21524 heap_segment_plan_allocated (nseg) =
21525 generation_allocation_pointer (gen);
21526 //switch allocation segment
21527 nseg = heap_segment_next_rw (nseg);
21528 generation_allocation_segment (gen) = nseg;
21529 //reset the allocation pointer and limits
21530 generation_allocation_pointer (gen) =
21531 heap_segment_mem (nseg);
21534 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21535 pinned_len (m) = plug - generation_allocation_pointer (gen);
21536 generation_allocation_pointer (gen) = plug + len;
21539 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21540 generation_allocation_pointer (gen) = 0;
21541 generation_allocation_limit (gen) = 0;
21546 void gc_heap::compact_loh()
21548 assert (should_compact_loh());
21550 generation* gen = large_object_generation;
21551 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21552 PREFIX_ASSUME(start_seg != NULL);
21553 heap_segment* seg = start_seg;
21554 heap_segment* prev_seg = 0;
21555 uint8_t* o = generation_allocation_start (gen);
21557 //Skip the generation gap object
21558 o = o + AlignQword (size (o));
21559 // We don't need to ever realloc gen3 start so don't touch it.
21560 uint8_t* free_space_start = o;
21561 uint8_t* free_space_end = o;
21562 generation_allocator (gen)->clear();
21563 generation_free_list_space (gen) = 0;
21564 generation_free_obj_space (gen) = 0;
21566 loh_pinned_queue_bos = 0;
21570 if (o >= heap_segment_allocated (seg))
21572 heap_segment* next_seg = heap_segment_next (seg);
21574 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21575 (seg != start_seg) && !heap_segment_read_only_p (seg))
21577 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21579 heap_segment_next (prev_seg) = next_seg;
21580 heap_segment_next (seg) = freeable_large_heap_segment;
21581 freeable_large_heap_segment = seg;
21585 if (!heap_segment_read_only_p (seg))
21587 // We grew the segment to accommodate allocations.
21588 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21590 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21592 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21596 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21597 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21598 decommit_heap_segment_pages (seg, 0);
21599 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21601 heap_segment_allocated (seg),
21602 heap_segment_used (seg),
21603 heap_segment_committed (seg)));
21604 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21605 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21615 o = heap_segment_mem (seg);
21621 free_space_end = o;
21622 size_t size = AlignQword (size (o));
21625 uint8_t* reloc = o;
21630 // We are relying on the fact the pinned objects are always looked at in the same order
21631 // in plan phase and in compact phase.
21632 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21633 uint8_t* plug = pinned_plug (m);
21634 assert (plug == o);
21636 loh_pad = pinned_len (m);
21641 loh_pad = AlignQword (loh_padding_obj_size);
21643 reloc += loh_node_relocation_distance (o);
21644 gcmemcopy (reloc, o, size, TRUE);
21647 thread_gap ((reloc - loh_pad), loh_pad, gen);
21650 free_space_start = o;
21651 if (o < heap_segment_allocated (seg))
21653 assert (!marked (o));
21658 while (o < heap_segment_allocated (seg) && !marked (o))
21660 o = o + AlignQword (size (o));
21665 assert (loh_pinned_plug_que_empty_p());
21667 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21668 generation_size (max_generation + 1),
21669 generation_free_list_space (gen),
21670 generation_free_obj_space (gen)));
21673 void gc_heap::relocate_in_loh_compact()
21675 generation* gen = large_object_generation;
21676 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21677 uint8_t* o = generation_allocation_start (gen);
21679 //Skip the generation gap object
21680 o = o + AlignQword (size (o));
21682 relocate_args args;
21684 args.high = gc_high;
21685 args.last_plug = 0;
21689 if (o >= heap_segment_allocated (seg))
21691 seg = heap_segment_next (seg);
21697 o = heap_segment_mem (seg);
21702 size_t size = AlignQword (size (o));
21704 check_class_object_demotion (o);
21705 if (contain_pointers (o))
21707 go_through_object_nostart (method_table (o), o, size(o), pval,
21709 reloc_survivor_helper (pval);
21714 if (o < heap_segment_allocated (seg))
21716 assert (!marked (o));
21721 while (o < heap_segment_allocated (seg) && !marked (o))
21723 o = o + AlignQword (size (o));
21728 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21729 generation_size (max_generation + 1),
21730 generation_free_list_space (gen),
21731 generation_free_obj_space (gen)));
21734 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21736 generation* gen = large_object_generation;
21737 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21738 uint8_t* o = generation_allocation_start (gen);
21740 //Skip the generation gap object
21741 o = o + AlignQword (size (o));
21745 if (o >= heap_segment_allocated (seg))
21747 seg = heap_segment_next (seg);
21753 o = heap_segment_mem (seg);
21758 size_t size = AlignQword (size (o));
21760 ptrdiff_t reloc = loh_node_relocation_distance (o);
21762 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21764 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21767 if (o < heap_segment_allocated (seg))
21769 assert (!marked (o));
21774 while (o < heap_segment_allocated (seg) && !marked (o))
21776 o = o + AlignQword (size (o));
21782 BOOL gc_heap::loh_object_p (uint8_t* o)
21784 #ifdef MULTIPLE_HEAPS
21785 gc_heap* hp = gc_heap::g_heaps [0];
21786 int brick_entry = hp->brick_table[hp->brick_of (o)];
21787 #else //MULTIPLE_HEAPS
21788 int brick_entry = brick_table[brick_of (o)];
21789 #endif //MULTIPLE_HEAPS
21791 return (brick_entry == 0);
21793 #endif //FEATURE_LOH_COMPACTION
21795 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21796 BOOL& last_pinned_plug_p,
21797 BOOL& pinned_plug_p,
21799 size_t& artificial_pinned_size)
21801 last_npinned_plug_p = FALSE;
21802 last_pinned_plug_p = TRUE;
21803 pinned_plug_p = TRUE;
21804 artificial_pinned_size = ps;
21807 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21808 // plugs are always interleaved.
21809 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21811 BOOL& last_npinned_plug_p,
21812 BOOL& last_pinned_plug_p,
21813 uint8_t*& last_pinned_plug,
21814 BOOL& pinned_plug_p,
21815 uint8_t* last_object_in_last_plug,
21816 BOOL& merge_with_last_pin_p,
21817 // this is only for verification purpose
21818 size_t last_plug_len)
21820 UNREFERENCED_PARAMETER(last_plug_len);
21822 if (!last_npinned_plug_p && !last_pinned_plug_p)
21824 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21825 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21826 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21827 set_gap_size (plug_start, plug_start - plug_end);
21830 if (pinned (plug_start))
21832 BOOL save_pre_plug_info_p = FALSE;
21834 if (last_npinned_plug_p || last_pinned_plug_p)
21836 //if (last_plug_len == Align (min_obj_size))
21838 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21839 // GCToOSInterface::DebugBreak();
21841 save_pre_plug_info_p = TRUE;
21844 pinned_plug_p = TRUE;
21845 last_npinned_plug_p = FALSE;
21847 if (last_pinned_plug_p)
21849 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21850 merge_with_last_pin_p = TRUE;
21854 last_pinned_plug_p = TRUE;
21855 last_pinned_plug = plug_start;
21857 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21859 if (save_pre_plug_info_p)
21861 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21867 if (last_pinned_plug_p)
21869 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21871 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21872 // GCToOSInterface::DebugBreak();
21875 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21876 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21878 verify_pins_with_post_plug_info("after saving post plug info");
21880 last_npinned_plug_p = TRUE;
21881 last_pinned_plug_p = FALSE;
21885 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21887 #ifdef GC_CONFIG_DRIVEN
21888 (interesting_data_per_gc[idp])++;
21890 UNREFERENCED_PARAMETER(idp);
21891 #endif //GC_CONFIG_DRIVEN
21895 #pragma warning(push)
21896 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21898 void gc_heap::plan_phase (int condemned_gen_number)
21900 size_t old_gen2_allocated = 0;
21901 size_t old_gen2_size = 0;
21903 if (condemned_gen_number == (max_generation - 1))
21905 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21906 old_gen2_size = generation_size (max_generation);
21909 assert (settings.concurrent == FALSE);
21911 // %type% category = quote (plan);
21915 start = GetCycleCount32();
21918 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21919 condemned_gen_number, settings.promotion ? 1 : 0));
21921 generation* condemned_gen1 = generation_of (condemned_gen_number);
21924 BOOL use_mark_list = FALSE;
21925 uint8_t** mark_list_next = &mark_list[0];
21926 #ifdef GC_CONFIG_DRIVEN
21927 dprintf (3, ("total number of marked objects: %Id (%Id)",
21928 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21930 if (mark_list_index >= (mark_list_end + 1))
21931 mark_list_index = mark_list_end + 1;
21933 dprintf (3, ("mark_list length: %Id",
21934 (mark_list_index - &mark_list[0])));
21935 #endif //GC_CONFIG_DRIVEN
21937 if ((condemned_gen_number < max_generation) &&
21938 (mark_list_index <= mark_list_end)
21939 #ifdef BACKGROUND_GC
21940 && (!recursive_gc_sync::background_running_p())
21941 #endif //BACKGROUND_GC
21944 #ifndef MULTIPLE_HEAPS
21945 _sort (&mark_list[0], mark_list_index-1, 0);
21946 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21947 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21948 #endif //!MULTIPLE_HEAPS
21949 use_mark_list = TRUE;
21950 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21954 dprintf (3, ("mark_list not used"));
21959 #ifdef FEATURE_BASICFREEZE
21960 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21961 ro_segments_in_range)
21963 sweep_ro_segments (generation_start_segment (condemned_gen1));
21965 #endif // FEATURE_BASICFREEZE
21967 #ifndef MULTIPLE_HEAPS
21968 if (shigh != (uint8_t*)0)
21970 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21972 PREFIX_ASSUME(seg != NULL);
21974 heap_segment* fseg = seg;
21977 if (slow > heap_segment_mem (seg) &&
21978 slow < heap_segment_reserved (seg))
21982 uint8_t* o = generation_allocation_start (condemned_gen1) +
21983 Align (size (generation_allocation_start (condemned_gen1)));
21986 assert ((slow - o) >= (int)Align (min_obj_size));
21987 #ifdef BACKGROUND_GC
21988 if (current_c_gc_state == c_gc_state_marking)
21990 bgc_clear_batch_mark_array_bits (o, slow);
21992 #endif //BACKGROUND_GC
21993 make_unused_array (o, slow - o);
21998 assert (condemned_gen_number == max_generation);
21999 make_unused_array (heap_segment_mem (seg),
22000 slow - heap_segment_mem (seg));
22003 if (in_range_for_segment (shigh, seg))
22005 #ifdef BACKGROUND_GC
22006 if (current_c_gc_state == c_gc_state_marking)
22008 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
22010 #endif //BACKGROUND_GC
22011 heap_segment_allocated (seg) = shigh + Align (size (shigh));
22013 // test if the segment is in the range of [slow, shigh]
22014 if (!((heap_segment_reserved (seg) >= slow) &&
22015 (heap_segment_mem (seg) <= shigh)))
22017 // shorten it to minimum
22018 heap_segment_allocated (seg) = heap_segment_mem (seg);
22020 seg = heap_segment_next_rw (seg);
22025 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
22027 PREFIX_ASSUME(seg != NULL);
22029 heap_segment* sseg = seg;
22032 // shorten it to minimum
22035 // no survivors make all generations look empty
22036 uint8_t* o = generation_allocation_start (condemned_gen1) +
22037 Align (size (generation_allocation_start (condemned_gen1)));
22038 #ifdef BACKGROUND_GC
22039 if (current_c_gc_state == c_gc_state_marking)
22041 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
22043 #endif //BACKGROUND_GC
22044 heap_segment_allocated (seg) = o;
22048 assert (condemned_gen_number == max_generation);
22049 #ifdef BACKGROUND_GC
22050 if (current_c_gc_state == c_gc_state_marking)
22052 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
22054 #endif //BACKGROUND_GC
22055 heap_segment_allocated (seg) = heap_segment_mem (seg);
22057 seg = heap_segment_next_rw (seg);
22061 #endif //MULTIPLE_HEAPS
22063 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
22065 PREFIX_ASSUME(seg1 != NULL);
22067 uint8_t* end = heap_segment_allocated (seg1);
22068 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
22069 uint8_t* x = first_condemned_address;
22071 assert (!marked (x));
22072 uint8_t* plug_end = x;
22074 size_t sequence_number = 0;
22075 uint8_t* last_node = 0;
22076 size_t current_brick = brick_of (x);
22077 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
22078 (settings.promotion == FALSE));
22079 int active_old_gen_number = condemned_gen_number;
22080 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
22081 (1 + condemned_gen_number));
22082 generation* older_gen = 0;
22083 generation* consing_gen = condemned_gen1;
22084 alloc_list r_free_list [MAX_BUCKET_COUNT];
22086 size_t r_free_list_space = 0;
22087 size_t r_free_obj_space = 0;
22088 size_t r_older_gen_free_list_allocated = 0;
22089 size_t r_older_gen_condemned_allocated = 0;
22090 size_t r_older_gen_end_seg_allocated = 0;
22091 uint8_t* r_allocation_pointer = 0;
22092 uint8_t* r_allocation_limit = 0;
22093 uint8_t* r_allocation_start_region = 0;
22094 heap_segment* r_allocation_segment = 0;
22095 #ifdef FREE_USAGE_STATS
22096 size_t r_older_gen_free_space[NUM_GEN_POWER2];
22097 #endif //FREE_USAGE_STATS
22099 if ((condemned_gen_number < max_generation))
22101 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
22102 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
22104 r_free_list_space = generation_free_list_space (older_gen);
22105 r_free_obj_space = generation_free_obj_space (older_gen);
22106 #ifdef FREE_USAGE_STATS
22107 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
22108 #endif //FREE_USAGE_STATS
22109 generation_allocate_end_seg_p (older_gen) = FALSE;
22110 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
22111 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
22112 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
22113 r_allocation_limit = generation_allocation_limit (older_gen);
22114 r_allocation_pointer = generation_allocation_pointer (older_gen);
22115 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
22116 r_allocation_segment = generation_allocation_segment (older_gen);
22117 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
22119 PREFIX_ASSUME(start_seg != NULL);
22121 if (start_seg != ephemeral_heap_segment)
22123 assert (condemned_gen_number == (max_generation - 1));
22124 while (start_seg && (start_seg != ephemeral_heap_segment))
22126 assert (heap_segment_allocated (start_seg) >=
22127 heap_segment_mem (start_seg));
22128 assert (heap_segment_allocated (start_seg) <=
22129 heap_segment_reserved (start_seg));
22130 heap_segment_plan_allocated (start_seg) =
22131 heap_segment_allocated (start_seg);
22132 start_seg = heap_segment_next_rw (start_seg);
22137 //reset all of the segment allocated sizes
22139 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
22141 PREFIX_ASSUME(seg2 != NULL);
22145 heap_segment_plan_allocated (seg2) =
22146 heap_segment_mem (seg2);
22147 seg2 = heap_segment_next_rw (seg2);
22150 int condemned_gn = condemned_gen_number;
22152 int bottom_gen = 0;
22153 init_free_and_plug();
22155 while (condemned_gn >= bottom_gen)
22157 generation* condemned_gen2 = generation_of (condemned_gn);
22158 generation_allocator (condemned_gen2)->clear();
22159 generation_free_list_space (condemned_gen2) = 0;
22160 generation_free_obj_space (condemned_gen2) = 0;
22161 generation_allocation_size (condemned_gen2) = 0;
22162 generation_condemned_allocated (condemned_gen2) = 0;
22163 generation_pinned_allocated (condemned_gen2) = 0;
22164 generation_free_list_allocated(condemned_gen2) = 0;
22165 generation_end_seg_allocated (condemned_gen2) = 0;
22166 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
22167 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
22168 #ifdef FREE_USAGE_STATS
22169 generation_pinned_free_obj_space (condemned_gen2) = 0;
22170 generation_allocated_in_pinned_free (condemned_gen2) = 0;
22171 generation_allocated_since_last_pin (condemned_gen2) = 0;
22172 #endif //FREE_USAGE_STATS
22173 generation_plan_allocation_start (condemned_gen2) = 0;
22174 generation_allocation_segment (condemned_gen2) =
22175 heap_segment_rw (generation_start_segment (condemned_gen2));
22177 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
22179 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
22181 generation_allocation_pointer (condemned_gen2) =
22182 heap_segment_mem (generation_allocation_segment (condemned_gen2));
22186 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
22189 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22190 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22195 BOOL allocate_first_generation_start = FALSE;
22197 if (allocate_in_condemned)
22199 allocate_first_generation_start = TRUE;
22202 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22204 demotion_low = MAX_PTR;
22205 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
22207 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
22208 // from gen1. They should get promoted to gen2.
22209 demote_gen1_p = !(settings.promotion &&
22210 (settings.condemned_generation == (max_generation - 1)) &&
22211 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
22213 total_ephemeral_size = 0;
22215 print_free_and_plug ("BP");
22217 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22219 generation* temp_gen = generation_of (gen_idx);
22221 dprintf (2, ("gen%d start %Ix, plan start %Ix",
22223 generation_allocation_start (temp_gen),
22224 generation_plan_allocation_start (temp_gen)));
22227 BOOL fire_pinned_plug_events_p = EVENT_ENABLED(PinPlugAtGCTime);
22228 size_t last_plug_len = 0;
22235 assert (heap_segment_allocated (seg1) == end);
22236 heap_segment_allocated (seg1) = plug_end;
22238 current_brick = update_brick_table (tree, current_brick, x, plug_end);
22239 dprintf (3, ("end of seg: new tree, sequence# 0"));
22240 sequence_number = 0;
22243 if (heap_segment_next_rw (seg1))
22245 seg1 = heap_segment_next_rw (seg1);
22246 end = heap_segment_allocated (seg1);
22247 plug_end = x = heap_segment_mem (seg1);
22248 current_brick = brick_of (x);
22249 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22258 BOOL last_npinned_plug_p = FALSE;
22259 BOOL last_pinned_plug_p = FALSE;
22261 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
22262 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
22263 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
22264 uint8_t* last_pinned_plug = 0;
22265 size_t num_pinned_plugs_in_plug = 0;
22267 uint8_t* last_object_in_plug = 0;
22269 while ((x < end) && marked (x))
22271 uint8_t* plug_start = x;
22272 uint8_t* saved_plug_end = plug_end;
22273 BOOL pinned_plug_p = FALSE;
22274 BOOL npin_before_pin_p = FALSE;
22275 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
22276 uint8_t* saved_last_object_in_plug = last_object_in_plug;
22277 BOOL merge_with_last_pin_p = FALSE;
22279 size_t added_pinning_size = 0;
22280 size_t artificial_pinned_size = 0;
22282 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
22283 last_pinned_plug, pinned_plug_p, last_object_in_plug,
22284 merge_with_last_pin_p, last_plug_len);
22286 #ifdef FEATURE_STRUCTALIGN
22287 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
22288 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
22289 #endif // FEATURE_STRUCTALIGN
22293 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
22300 #ifdef FEATURE_STRUCTALIGN
22303 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
22304 if (obj_requiredAlignment > requiredAlignment)
22306 requiredAlignment = obj_requiredAlignment;
22307 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
22310 #endif // FEATURE_STRUCTALIGN
22314 dprintf(4, ("+%Ix+", (size_t)xl));
22315 assert ((size (xl) > 0));
22316 assert ((size (xl) <= loh_size_threshold));
22318 last_object_in_plug = xl;
22320 xl = xl + Align (size (xl));
22324 BOOL next_object_marked_p = ((xl < end) && marked (xl));
22328 // If it is pinned we need to extend to the next marked object as we can't use part of
22329 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
22330 // references but for now I am just using the next non pinned object for that).
22331 if (next_object_marked_p)
22334 last_object_in_plug = xl;
22335 size_t extra_size = Align (size (xl));
22336 xl = xl + extra_size;
22337 added_pinning_size = extra_size;
22342 if (next_object_marked_p)
22343 npin_before_pin_p = TRUE;
22346 assert (xl <= end);
22349 dprintf (3, ( "%Ix[", (size_t)x));
22351 size_t ps = plug_end - plug_start;
22352 last_plug_len = ps;
22353 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
22354 uint8_t* new_address = 0;
22356 if (!pinned_plug_p)
22358 if (allocate_in_condemned &&
22359 (settings.condemned_generation == max_generation) &&
22360 (ps > OS_PAGE_SIZE))
22362 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
22363 //reloc should >=0 except when we relocate
22364 //across segments and the dest seg is higher then the src
22366 if ((ps > (8*OS_PAGE_SIZE)) &&
22368 ((size_t)reloc < (ps/16)))
22370 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
22371 (size_t)plug_start, reloc));
22372 // The last plug couldn't have been a npinned plug or it would have
22373 // included this plug.
22374 assert (!saved_last_npinned_plug_p);
22376 if (last_pinned_plug)
22378 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
22379 merge_with_last_pin_p = TRUE;
22383 enque_pinned_plug (plug_start, FALSE, 0);
22384 last_pinned_plug = plug_start;
22387 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22388 ps, artificial_pinned_size);
22393 if (allocate_first_generation_start)
22395 allocate_first_generation_start = FALSE;
22396 plan_generation_start (condemned_gen1, consing_gen, plug_start);
22397 assert (generation_plan_allocation_start (condemned_gen1));
22400 if (seg1 == ephemeral_heap_segment)
22402 process_ephemeral_boundaries (plug_start, active_new_gen_number,
22403 active_old_gen_number,
22405 allocate_in_condemned);
22408 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
22410 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
22411 dd_survived_size (dd_active_old) += ps;
22413 BOOL convert_to_pinned_p = FALSE;
22415 if (!pinned_plug_p)
22417 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
22418 dd_num_npinned_plugs (dd_active_old)++;
22419 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
22421 add_gen_plug (active_old_gen_number, ps);
22423 if (allocate_in_condemned)
22425 verify_pins_with_post_plug_info("before aic");
22428 allocate_in_condemned_generations (consing_gen,
22430 active_old_gen_number,
22432 &convert_to_pinned_p,
22433 (npin_before_pin_p ? plug_end : 0),
22435 #endif //SHORT_PLUGS
22436 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22437 verify_pins_with_post_plug_info("after aic");
22441 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
22443 if (new_address != 0)
22445 if (settings.condemned_generation == (max_generation - 1))
22447 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
22448 plug_start, plug_end,
22449 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
22450 (size_t)(plug_end - plug_start)));
22455 if (generation_allocator(older_gen)->discard_if_no_fit_p())
22457 allocate_in_condemned = TRUE;
22460 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
22462 &convert_to_pinned_p,
22463 (npin_before_pin_p ? plug_end : 0),
22465 #endif //SHORT_PLUGS
22466 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22470 if (convert_to_pinned_p)
22472 assert (last_npinned_plug_p != FALSE);
22473 assert (last_pinned_plug_p == FALSE);
22474 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22475 ps, artificial_pinned_size);
22476 enque_pinned_plug (plug_start, FALSE, 0);
22477 last_pinned_plug = plug_start;
22483 //verify that we are at then end of the ephemeral segment
22484 assert (generation_allocation_segment (consing_gen) ==
22485 ephemeral_heap_segment);
22486 //verify that we are near the end
22487 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
22488 heap_segment_allocated (ephemeral_heap_segment));
22489 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
22490 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
22494 #ifdef SIMPLE_DPRINTF
22495 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
22496 (size_t)(node_gap_size (plug_start)),
22497 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
22498 (size_t)new_address + ps, ps,
22499 (is_plug_padded (plug_start) ? 1 : 0)));
22500 #endif //SIMPLE_DPRINTF
22503 if (is_plug_padded (plug_start))
22505 dprintf (3, ("%Ix was padded", plug_start));
22506 dd_padding_size (dd_active_old) += Align (min_obj_size);
22508 #endif //SHORT_PLUGS
22515 if (fire_pinned_plug_events_p)
22517 FIRE_EVENT(PinPlugAtGCTime, plug_start, plug_end,
22518 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)));
22521 if (merge_with_last_pin_p)
22523 merge_with_last_pinned_plug (last_pinned_plug, ps);
22527 assert (last_pinned_plug == plug_start);
22528 set_pinned_info (plug_start, ps, consing_gen);
22531 new_address = plug_start;
22533 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22534 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22535 (size_t)plug_end, ps,
22536 (merge_with_last_pin_p ? 1 : 0)));
22538 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22539 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22540 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22541 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22543 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22545 last_gen1_pin_end = plug_end;
22550 // detect forward allocation in the same segment
22551 assert (!((new_address > plug_start) &&
22552 (new_address < heap_segment_reserved (seg1))));
22555 if (!merge_with_last_pin_p)
22557 if (current_brick != brick_of (plug_start))
22559 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22560 sequence_number = 0;
22564 set_node_relocation_distance (plug_start, (new_address - plug_start));
22565 if (last_node && (node_relocation_distance (last_node) ==
22566 (node_relocation_distance (plug_start) +
22567 (ptrdiff_t)node_gap_size (plug_start))))
22569 //dprintf(3,( " Lb"));
22570 dprintf (3, ("%Ix Lb", plug_start));
22571 set_node_left (plug_start);
22573 if (0 == sequence_number)
22575 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22579 verify_pins_with_post_plug_info("before insert node");
22581 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22582 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22583 last_node = plug_start;
22586 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22587 if (!pinned_plug_p)
22589 if (mark_stack_tos > 0)
22591 mark& m = mark_stack_array[mark_stack_tos - 1];
22592 if (m.has_post_plug_info())
22594 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22595 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22596 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22598 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22599 *current_plug_gap_start, *(current_plug_gap_start + 1),
22600 *(current_plug_gap_start + 2)));
22601 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22608 verify_pins_with_post_plug_info("after insert node");
22612 if (num_pinned_plugs_in_plug > 1)
22614 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22621 while ((mark_list_next < mark_list_index) &&
22622 (*mark_list_next <= x))
22626 if ((mark_list_next < mark_list_index)
22627 #ifdef MULTIPLE_HEAPS
22628 && (*mark_list_next < end) //for multiple segments
22629 #endif //MULTIPLE_HEAPS
22631 x = *mark_list_next;
22639 #ifdef BACKGROUND_GC
22640 if (current_c_gc_state == c_gc_state_marking)
22642 assert (recursive_gc_sync::background_running_p());
22643 while ((xl < end) && !marked (xl))
22645 dprintf (4, ("-%Ix-", (size_t)xl));
22646 assert ((size (xl) > 0));
22647 background_object_marked (xl, TRUE);
22648 xl = xl + Align (size (xl));
22653 #endif //BACKGROUND_GC
22655 while ((xl < end) && !marked (xl))
22657 dprintf (4, ("-%Ix-", (size_t)xl));
22658 assert ((size (xl) > 0));
22659 xl = xl + Align (size (xl));
22663 assert (xl <= end);
22669 while (!pinned_plug_que_empty_p())
22671 if (settings.promotion)
22673 uint8_t* pplug = pinned_plug (oldest_pin());
22674 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22676 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22677 //allocate all of the generation gaps
22678 while (active_new_gen_number > 0)
22680 active_new_gen_number--;
22682 if (active_new_gen_number == (max_generation - 1))
22684 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22685 if (!demote_gen1_p)
22686 advance_pins_for_demotion (consing_gen);
22689 generation* gen = generation_of (active_new_gen_number);
22690 plan_generation_start (gen, consing_gen, 0);
22692 if (demotion_low == MAX_PTR)
22694 demotion_low = pplug;
22695 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22698 dprintf (2, ("(%d)gen%d plan start: %Ix",
22699 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22700 assert (generation_plan_allocation_start (gen));
22705 if (pinned_plug_que_empty_p())
22708 size_t entry = deque_pinned_plug();
22709 mark* m = pinned_plug_of (entry);
22710 uint8_t* plug = pinned_plug (m);
22711 size_t len = pinned_len (m);
22713 // detect pinned block in different segment (later) than
22714 // allocation segment
22715 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22717 while ((plug < generation_allocation_pointer (consing_gen)) ||
22718 (plug >= heap_segment_allocated (nseg)))
22720 assert ((plug < heap_segment_mem (nseg)) ||
22721 (plug > heap_segment_reserved (nseg)));
22722 //adjust the end of the segment to be the end of the plug
22723 assert (generation_allocation_pointer (consing_gen)>=
22724 heap_segment_mem (nseg));
22725 assert (generation_allocation_pointer (consing_gen)<=
22726 heap_segment_committed (nseg));
22728 heap_segment_plan_allocated (nseg) =
22729 generation_allocation_pointer (consing_gen);
22730 //switch allocation segment
22731 nseg = heap_segment_next_rw (nseg);
22732 generation_allocation_segment (consing_gen) = nseg;
22733 //reset the allocation pointer and limits
22734 generation_allocation_pointer (consing_gen) =
22735 heap_segment_mem (nseg);
22738 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22739 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22740 (size_t)(brick_table[brick_of (plug)])));
22742 generation_allocation_pointer (consing_gen) = plug + len;
22743 generation_allocation_limit (consing_gen) =
22744 generation_allocation_pointer (consing_gen);
22745 //Add the size of the pinned plug to the right pinned allocations
22746 //find out which gen this pinned plug came from
22747 int frgn = object_gennum (plug);
22748 if ((frgn != (int)max_generation) && settings.promotion)
22750 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22755 plan_generation_starts (consing_gen);
22756 print_free_and_plug ("AP");
22759 #ifdef SIMPLE_DPRINTF
22760 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22762 generation* temp_gen = generation_of (gen_idx);
22763 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22765 int added_pinning_ratio = 0;
22766 int artificial_pinned_ratio = 0;
22768 if (dd_pinned_survived_size (temp_dd) != 0)
22770 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22771 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22774 size_t padding_size =
22776 dd_padding_size (temp_dd);
22779 #endif //SHORT_PLUGS
22780 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22782 generation_allocation_start (temp_gen),
22783 generation_plan_allocation_start (temp_gen),
22784 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22785 generation_allocation_size (temp_gen),
22786 generation_pinned_allocation_compact_size (temp_gen),
22787 generation_pinned_allocation_sweep_size (temp_gen),
22788 dd_survived_size (temp_dd),
22789 dd_pinned_survived_size (temp_dd),
22790 added_pinning_ratio,
22791 artificial_pinned_ratio,
22792 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22795 #endif //SIMPLE_DPRINTF
22798 if (settings.condemned_generation == (max_generation - 1 ))
22800 size_t plan_gen2_size = generation_plan_size (max_generation);
22801 size_t growth = plan_gen2_size - old_gen2_size;
22803 generation* older_gen = generation_of (settings.condemned_generation + 1);
22804 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22805 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22806 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22807 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22811 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, condemned alloc: %Id",
22812 growth, end_seg_allocated, condemned_allocated));
22814 maxgen_size_inc_p = true;
22818 dprintf (2, ("gen2 shrank %Id (end seg alloc: %Id, , condemned alloc: %Id, gen1 c alloc: %Id",
22819 (old_gen2_size - plan_gen2_size), end_seg_allocated, condemned_allocated,
22820 generation_condemned_allocated (generation_of (max_generation - 1))));
22823 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22824 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22825 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22826 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22828 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected)",
22829 free_list_allocated, rejected_free_space));
22831 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22832 maxgen_size_info->free_list_allocated = free_list_allocated;
22833 maxgen_size_info->free_list_rejected = rejected_free_space;
22834 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22835 maxgen_size_info->condemned_allocated = condemned_allocated;
22836 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22837 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22839 #ifdef FREE_USAGE_STATS
22840 int free_list_efficiency = 0;
22841 if ((free_list_allocated + rejected_free_space) != 0)
22842 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22844 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22846 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22847 older_gen->gen_num,
22848 free_list_efficiency, running_free_list_efficiency));
22850 dprintf (1, ("gen2 free list change"));
22851 for (int j = 0; j < NUM_GEN_POWER2; j++)
22853 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22856 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22857 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22858 (generation_of(max_generation - 1))->gen_plugs[j]));
22860 #endif //FREE_USAGE_STATS
22863 size_t fragmentation =
22864 generation_fragmentation (generation_of (condemned_gen_number),
22866 heap_segment_allocated (ephemeral_heap_segment));
22868 dprintf (2,("Fragmentation: %Id", fragmentation));
22869 dprintf (2,("---- End of Plan phase ----"));
22872 finish = GetCycleCount32();
22873 plan_time = finish - start;
22876 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22877 assert(IsGCInProgress());
22879 BOOL should_expand = FALSE;
22880 BOOL should_compact= FALSE;
22881 ephemeral_promotion = FALSE;
22884 if ((!settings.concurrent) &&
22885 !provisional_mode_triggered &&
22886 ((condemned_gen_number < max_generation) &&
22887 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22889 dprintf (GTC_LOG, ("gen0 reduction count is %d, condemning %d, mem load %d",
22890 settings.gen0_reduction_count,
22891 condemned_gen_number,
22892 settings.entry_memory_load));
22893 should_compact = TRUE;
22895 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22896 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22898 if ((condemned_gen_number >= (max_generation - 1)) &&
22899 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22901 dprintf (GTC_LOG, ("Not enough space for all ephemeral generations with compaction"));
22902 should_expand = TRUE;
22908 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22913 #ifdef FEATURE_LOH_COMPACTION
22914 loh_compacted_p = FALSE;
22915 #endif //FEATURE_LOH_COMPACTION
22917 if (condemned_gen_number == max_generation)
22919 #ifdef FEATURE_LOH_COMPACTION
22920 if (settings.loh_compaction)
22924 should_compact = TRUE;
22925 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22926 loh_compacted_p = TRUE;
22931 if ((heap_number == 0) && (loh_pinned_queue))
22933 loh_pinned_queue_decay--;
22935 if (!loh_pinned_queue_decay)
22937 delete loh_pinned_queue;
22938 loh_pinned_queue = 0;
22943 if (!loh_compacted_p)
22944 #endif //FEATURE_LOH_COMPACTION
22946 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22947 sweep_large_objects();
22952 settings.loh_compaction = FALSE;
22955 #ifdef MULTIPLE_HEAPS
22957 new_heap_segment = NULL;
22959 if (should_compact && should_expand)
22960 gc_policy = policy_expand;
22961 else if (should_compact)
22962 gc_policy = policy_compact;
22964 gc_policy = policy_sweep;
22966 //vote for result of should_compact
22967 dprintf (3, ("Joining for compaction decision"));
22968 gc_t_join.join(this, gc_join_decide_on_compaction);
22969 if (gc_t_join.joined())
22971 //safe place to delete large heap segments
22972 if (condemned_gen_number == max_generation)
22974 for (int i = 0; i < n_heaps; i++)
22976 g_heaps [i]->rearrange_large_heap_segments ();
22980 if (maxgen_size_inc_p && provisional_mode_triggered)
22982 pm_trigger_full_gc = true;
22983 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
22987 settings.demotion = FALSE;
22988 int pol_max = policy_sweep;
22989 #ifdef GC_CONFIG_DRIVEN
22990 BOOL is_compaction_mandatory = FALSE;
22991 #endif //GC_CONFIG_DRIVEN
22994 for (i = 0; i < n_heaps; i++)
22996 if (pol_max < g_heaps[i]->gc_policy)
22997 pol_max = policy_compact;
22998 // set the demotion flag is any of the heap has demotion
22999 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
23001 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
23002 settings.demotion = TRUE;
23005 #ifdef GC_CONFIG_DRIVEN
23006 if (!is_compaction_mandatory)
23008 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
23009 if (compact_reason >= 0)
23011 if (gc_heap_compact_reason_mandatory_p[compact_reason])
23012 is_compaction_mandatory = TRUE;
23015 #endif //GC_CONFIG_DRIVEN
23018 #ifdef GC_CONFIG_DRIVEN
23019 if (!is_compaction_mandatory)
23021 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
23022 // Note that we may want to change this to only checking every so often instead of every single GC.
23023 if (should_do_sweeping_gc (pol_max >= policy_compact))
23025 pol_max = policy_sweep;
23029 if (pol_max == policy_sweep)
23030 pol_max = policy_compact;
23033 #endif //GC_CONFIG_DRIVEN
23035 for (i = 0; i < n_heaps; i++)
23037 if (pol_max > g_heaps[i]->gc_policy)
23038 g_heaps[i]->gc_policy = pol_max;
23039 //get the segment while we are serialized
23040 if (g_heaps[i]->gc_policy == policy_expand)
23042 g_heaps[i]->new_heap_segment =
23043 g_heaps[i]->soh_get_segment_to_expand();
23044 if (!g_heaps[i]->new_heap_segment)
23046 set_expand_in_full_gc (condemned_gen_number);
23047 //we are out of memory, cancel the expansion
23048 g_heaps[i]->gc_policy = policy_compact;
23053 BOOL is_full_compacting_gc = FALSE;
23055 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
23057 full_gc_counts[gc_type_compacting]++;
23058 is_full_compacting_gc = TRUE;
23061 for (i = 0; i < n_heaps; i++)
23063 //copy the card and brick tables
23064 if (g_gc_card_table!= g_heaps[i]->card_table)
23066 g_heaps[i]->copy_brick_card_table();
23069 if (is_full_compacting_gc)
23071 g_heaps[i]->loh_alloc_since_cg = 0;
23076 //start all threads on the roots.
23077 dprintf(3, ("Starting all gc threads after compaction decision"));
23078 gc_t_join.restart();
23081 //reset the local variable accordingly
23082 should_compact = (gc_policy >= policy_compact);
23083 should_expand = (gc_policy >= policy_expand);
23085 #else //MULTIPLE_HEAPS
23087 //safe place to delete large heap segments
23088 if (condemned_gen_number == max_generation)
23090 rearrange_large_heap_segments ();
23093 if (maxgen_size_inc_p && provisional_mode_triggered)
23095 pm_trigger_full_gc = true;
23096 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
23100 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
23101 if (settings.demotion)
23102 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
23104 #ifdef GC_CONFIG_DRIVEN
23105 BOOL is_compaction_mandatory = FALSE;
23106 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
23107 if (compact_reason >= 0)
23108 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
23110 if (!is_compaction_mandatory)
23112 if (should_do_sweeping_gc (should_compact))
23113 should_compact = FALSE;
23115 should_compact = TRUE;
23117 #endif //GC_CONFIG_DRIVEN
23119 if (should_compact && (condemned_gen_number == max_generation))
23121 full_gc_counts[gc_type_compacting]++;
23122 loh_alloc_since_cg = 0;
23125 #endif //MULTIPLE_HEAPS
23127 if (!pm_trigger_full_gc && pm_stress_on && provisional_mode_triggered)
23129 if ((settings.condemned_generation == (max_generation - 1)) &&
23130 ((settings.gc_index % 5) == 0))
23132 pm_trigger_full_gc = true;
23136 if (settings.condemned_generation == (max_generation - 1))
23138 if (provisional_mode_triggered)
23142 should_expand = FALSE;
23143 dprintf (GTC_LOG, ("h%d in PM cannot expand", heap_number));
23147 if (pm_trigger_full_gc)
23149 should_compact = FALSE;
23150 dprintf (GTC_LOG, ("h%d PM doing sweeping", heap_number));
23154 if (should_compact)
23156 dprintf (2,( "**** Doing Compacting GC ****"));
23160 #ifndef MULTIPLE_HEAPS
23161 heap_segment* new_heap_segment = soh_get_segment_to_expand();
23162 #endif //!MULTIPLE_HEAPS
23163 if (new_heap_segment)
23165 consing_gen = expand_heap(condemned_gen_number,
23170 // If we couldn't get a new segment, or we were able to
23171 // reserve one but no space to commit, we couldn't
23173 if (ephemeral_heap_segment != new_heap_segment)
23175 set_expand_in_full_gc (condemned_gen_number);
23176 should_expand = FALSE;
23179 generation_allocation_limit (condemned_gen1) =
23180 generation_allocation_pointer (condemned_gen1);
23181 if ((condemned_gen_number < max_generation))
23183 generation_allocator (older_gen)->commit_alloc_list_changes();
23185 // Fix the allocation area of the older generation
23186 fix_older_allocation_area (older_gen);
23188 assert (generation_allocation_segment (consing_gen) ==
23189 ephemeral_heap_segment);
23191 GCToEEInterface::DiagWalkSurvivors(__this);
23193 relocate_phase (condemned_gen_number, first_condemned_address);
23194 compact_phase (condemned_gen_number, first_condemned_address,
23195 (!settings.demotion && settings.promotion));
23196 fix_generation_bounds (condemned_gen_number, consing_gen);
23197 assert (generation_allocation_limit (youngest_generation) ==
23198 generation_allocation_pointer (youngest_generation));
23199 if (condemned_gen_number >= (max_generation -1))
23201 #ifdef MULTIPLE_HEAPS
23202 // this needs be serialized just because we have one
23203 // segment_standby_list/seg_table for all heaps. We should make it at least
23204 // so that when hoarding is not on we don't need this join because
23205 // decommitting memory can take a long time.
23206 //must serialize on deleting segments
23207 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
23208 if (gc_t_join.joined())
23210 for (int i = 0; i < n_heaps; i++)
23212 g_heaps[i]->rearrange_heap_segments(TRUE);
23214 gc_t_join.restart();
23217 rearrange_heap_segments(TRUE);
23218 #endif //MULTIPLE_HEAPS
23222 //fix the start_segment for the ephemeral generations
23223 for (int i = 0; i < max_generation; i++)
23225 generation* gen = generation_of (i);
23226 generation_start_segment (gen) = ephemeral_heap_segment;
23227 generation_allocation_segment (gen) = ephemeral_heap_segment;
23233 #ifdef FEATURE_PREMORTEM_FINALIZATION
23234 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
23235 (!settings.demotion && settings.promotion));
23236 #endif // FEATURE_PREMORTEM_FINALIZATION
23238 #ifdef MULTIPLE_HEAPS
23239 dprintf(3, ("Joining after end of compaction"));
23240 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
23241 if (gc_t_join.joined())
23242 #endif //MULTIPLE_HEAPS
23244 #ifdef MULTIPLE_HEAPS
23245 //join all threads to make sure they are synchronized
23246 dprintf(3, ("Restarting after Promotion granted"));
23247 gc_t_join.restart();
23248 #endif //MULTIPLE_HEAPS
23252 sc.thread_number = heap_number;
23253 sc.promotion = FALSE;
23254 sc.concurrent = FALSE;
23255 // new generations bounds are set can call this guy
23256 if (settings.promotion && !settings.demotion)
23258 dprintf (2, ("Promoting EE roots for gen %d",
23259 condemned_gen_number));
23260 GCScan::GcPromotionsGranted(condemned_gen_number,
23261 max_generation, &sc);
23263 else if (settings.demotion)
23265 dprintf (2, ("Demoting EE roots for gen %d",
23266 condemned_gen_number));
23267 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
23272 gen0_big_free_spaces = 0;
23274 reset_pinned_queue_bos();
23275 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
23276 generation* gen = generation_of (gen_number);
23277 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
23278 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
23280 while (!pinned_plug_que_empty_p())
23282 mark* m = pinned_plug_of (deque_pinned_plug());
23283 size_t len = pinned_len (m);
23284 uint8_t* arr = (pinned_plug (m) - len);
23285 dprintf(3,("free [%Ix %Ix[ pin",
23286 (size_t)arr, (size_t)arr + len));
23289 assert (len >= Align (min_obj_size));
23290 make_unused_array (arr, len);
23291 // fix fully contained bricks + first one
23292 // if the array goes beyond the first brick
23293 size_t start_brick = brick_of (arr);
23294 size_t end_brick = brick_of (arr + len);
23295 if (end_brick != start_brick)
23298 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
23299 start_brick, end_brick, (size_t)arr));
23300 set_brick (start_brick,
23301 arr - brick_address (start_brick));
23302 size_t brick = start_brick+1;
23303 while (brick < end_brick)
23305 set_brick (brick, start_brick - brick);
23310 //when we take an old segment to make the new
23311 //ephemeral segment. we can have a bunch of
23312 //pinned plugs out of order going to the new ephemeral seg
23313 //and then the next plugs go back to max_generation
23314 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
23315 (heap_segment_reserved (ephemeral_heap_segment) > arr))
23318 while ((low <= arr) && (high > arr))
23321 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
23322 settings.demotion || !settings.promotion);
23323 dprintf (3, ("new free list generation %d", gen_number));
23325 gen = generation_of (gen_number);
23326 if (gen_number >= 1)
23327 low = generation_allocation_start (generation_of (gen_number-1));
23334 dprintf (3, ("new free list generation %d", max_generation));
23335 gen_number = max_generation;
23336 gen = generation_of (gen_number);
23339 dprintf(3,("threading it into generation %d", gen_number));
23340 thread_gap (arr, len, gen);
23341 add_gen_free (gen_number, len);
23347 for (int x = 0; x <= max_generation; x++)
23349 assert (generation_allocation_start (generation_of (x)));
23353 if (!settings.demotion && settings.promotion)
23355 //clear card for generation 1. generation 0 is empty
23356 clear_card_for_addresses (
23357 generation_allocation_start (generation_of (1)),
23358 generation_allocation_start (generation_of (0)));
23360 if (settings.promotion && !settings.demotion)
23362 uint8_t* start = generation_allocation_start (youngest_generation);
23363 MAYBE_UNUSED_VAR(start);
23364 assert (heap_segment_allocated (ephemeral_heap_segment) ==
23365 (start + Align (size (start))));
23370 //force promotion for sweep
23371 settings.promotion = TRUE;
23372 settings.compaction = FALSE;
23375 sc.thread_number = heap_number;
23376 sc.promotion = FALSE;
23377 sc.concurrent = FALSE;
23379 dprintf (2, ("**** Doing Mark and Sweep GC****"));
23381 if ((condemned_gen_number < max_generation))
23383 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
23384 generation_free_list_space (older_gen) = r_free_list_space;
23385 generation_free_obj_space (older_gen) = r_free_obj_space;
23386 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
23387 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
23388 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
23389 generation_allocation_limit (older_gen) = r_allocation_limit;
23390 generation_allocation_pointer (older_gen) = r_allocation_pointer;
23391 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
23392 generation_allocation_segment (older_gen) = r_allocation_segment;
23395 if ((condemned_gen_number < max_generation))
23397 // Fix the allocation area of the older generation
23398 fix_older_allocation_area (older_gen);
23401 GCToEEInterface::DiagWalkSurvivors(__this);
23403 gen0_big_free_spaces = 0;
23404 make_free_lists (condemned_gen_number);
23405 recover_saved_pinned_info();
23407 #ifdef FEATURE_PREMORTEM_FINALIZATION
23408 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
23409 #endif // FEATURE_PREMORTEM_FINALIZATION
23410 // MTHTS: leave single thread for HT processing on plan_phase
23411 #ifdef MULTIPLE_HEAPS
23412 dprintf(3, ("Joining after end of sweep"));
23413 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
23414 if (gc_t_join.joined())
23415 #endif //MULTIPLE_HEAPS
23417 GCScan::GcPromotionsGranted(condemned_gen_number,
23418 max_generation, &sc);
23419 if (condemned_gen_number >= (max_generation -1))
23421 #ifdef MULTIPLE_HEAPS
23422 for (int i = 0; i < n_heaps; i++)
23424 g_heaps[i]->rearrange_heap_segments(FALSE);
23427 rearrange_heap_segments(FALSE);
23428 #endif //MULTIPLE_HEAPS
23431 #ifdef MULTIPLE_HEAPS
23432 //join all threads to make sure they are synchronized
23433 dprintf(3, ("Restarting after Promotion granted"));
23434 gc_t_join.restart();
23435 #endif //MULTIPLE_HEAPS
23439 for (int x = 0; x <= max_generation; x++)
23441 assert (generation_allocation_start (generation_of (x)));
23445 //clear card for generation 1. generation 0 is empty
23446 clear_card_for_addresses (
23447 generation_allocation_start (generation_of (1)),
23448 generation_allocation_start (generation_of (0)));
23449 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
23450 (generation_allocation_start (youngest_generation) +
23451 Align (min_obj_size))));
23454 //verify_partial();
23457 #pragma warning(pop)
23461 /*****************************
23462 Called after compact phase to fix all generation gaps
23463 ********************************/
23464 void gc_heap::fix_generation_bounds (int condemned_gen_number,
23465 generation* consing_gen)
23467 UNREFERENCED_PARAMETER(consing_gen);
23469 assert (generation_allocation_segment (consing_gen) ==
23470 ephemeral_heap_segment);
23472 //assign the planned allocation start to the generation
23473 int gen_number = condemned_gen_number;
23474 int bottom_gen = 0;
23476 while (gen_number >= bottom_gen)
23478 generation* gen = generation_of (gen_number);
23479 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
23480 if ((gen_number < max_generation) && ephemeral_promotion)
23482 make_unused_array (saved_ephemeral_plan_start[gen_number],
23483 saved_ephemeral_plan_start_size[gen_number]);
23485 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
23486 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
23487 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
23490 #ifdef MULTIPLE_HEAPS
23491 if (ephemeral_promotion)
23493 //we are creating a generation fault. set the cards.
23494 // and we are only doing this for multiple heaps because in the single heap scenario the
23495 // new ephemeral generations will be empty and there'll be no need to set cards for the
23496 // old ephemeral generations that got promoted into max_generation.
23497 ptrdiff_t delta = 0;
23498 #ifdef SEG_MAPPING_TABLE
23499 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
23500 #else //SEG_MAPPING_TABLE
23501 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
23502 #endif //SEG_MAPPING_TABLE
23504 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
23505 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
23506 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
23507 while (card != end_card)
23513 #endif //MULTIPLE_HEAPS
23515 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
23516 //reset the allocated size
23517 uint8_t* start = generation_allocation_start (youngest_generation);
23518 MAYBE_UNUSED_VAR(start);
23519 if (settings.promotion && !settings.demotion)
23521 assert ((start + Align (size (start))) ==
23522 heap_segment_plan_allocated(ephemeral_heap_segment));
23525 heap_segment_allocated(ephemeral_heap_segment)=
23526 heap_segment_plan_allocated(ephemeral_heap_segment);
23530 uint8_t* gc_heap::generation_limit (int gen_number)
23532 if (settings.promotion)
23534 if (gen_number <= 1)
23535 return heap_segment_reserved (ephemeral_heap_segment);
23537 return generation_allocation_start (generation_of ((gen_number - 2)));
23541 if (gen_number <= 0)
23542 return heap_segment_reserved (ephemeral_heap_segment);
23544 return generation_allocation_start (generation_of ((gen_number - 1)));
23548 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
23550 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23551 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
23552 assert ((start + size) <=
23553 heap_segment_reserved (ephemeral_heap_segment));
23554 if ((start + size) >
23555 heap_segment_committed (ephemeral_heap_segment))
23557 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
23565 uint8_t* gc_heap::allocate_at_end (size_t size)
23567 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23568 size = Align (size);
23569 uint8_t* result = start;
23570 // only called to allocate a min obj so can't overflow here.
23571 assert ((start + size) <=
23572 heap_segment_reserved (ephemeral_heap_segment));
23573 //ensure_gap_allocation took care of it
23574 assert ((start + size) <=
23575 heap_segment_committed (ephemeral_heap_segment));
23576 heap_segment_allocated (ephemeral_heap_segment) += size;
23581 void gc_heap::make_free_lists (int condemned_gen_number)
23586 start = GetCycleCount32();
23589 //Promotion has to happen in sweep case.
23590 assert (settings.promotion);
23592 generation* condemned_gen = generation_of (condemned_gen_number);
23593 uint8_t* start_address = generation_allocation_start (condemned_gen);
23595 size_t current_brick = brick_of (start_address);
23596 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23598 PREFIX_ASSUME(current_heap_segment != NULL);
23600 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23601 size_t end_brick = brick_of (end_address-1);
23602 make_free_args args;
23603 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23604 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23606 (generation_limit (args.free_list_gen_number)));
23607 args.free_list_gen = generation_of (args.free_list_gen_number);
23608 args.highest_plug = 0;
23610 if ((start_address < end_address) ||
23611 (condemned_gen_number == max_generation))
23615 if ((current_brick > end_brick))
23617 if (args.current_gen_limit == MAX_PTR)
23619 //We had an empty segment
23620 //need to allocate the generation start
23622 generation* gen = generation_of (max_generation);
23624 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23626 PREFIX_ASSUME(start_seg != NULL);
23628 uint8_t* gap = heap_segment_mem (start_seg);
23630 generation_allocation_start (gen) = gap;
23631 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23632 make_unused_array (gap, Align (min_obj_size));
23633 reset_allocation_pointers (gen, gap);
23634 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23635 max_generation, (size_t)gap));
23636 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23638 if (heap_segment_next_rw (current_heap_segment))
23640 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23641 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23642 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23652 int brick_entry = brick_table [ current_brick ];
23653 if ((brick_entry >= 0))
23655 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23656 dprintf(3,("Fixing brick entry %Ix to %Ix",
23657 current_brick, (size_t)args.highest_plug));
23658 set_brick (current_brick,
23659 (args.highest_plug - brick_address (current_brick)));
23663 if ((brick_entry > -32768))
23667 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23668 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23670 assert ((brick_entry == -1));
23673 //init to -1 for faster find_first_object
23674 set_brick (current_brick, -1);
23682 int bottom_gen = 0;
23683 args.free_list_gen_number--;
23684 while (args.free_list_gen_number >= bottom_gen)
23687 generation* gen2 = generation_of (args.free_list_gen_number);
23688 gap = allocate_at_end (Align(min_obj_size));
23689 generation_allocation_start (gen2) = gap;
23690 reset_allocation_pointers (gen2, gap);
23691 dprintf(3,("Fixing generation start of %d to: %Ix",
23692 args.free_list_gen_number, (size_t)gap));
23693 PREFIX_ASSUME(gap != NULL);
23694 make_unused_array (gap, Align (min_obj_size));
23696 args.free_list_gen_number--;
23699 //reset the allocated size
23700 uint8_t* start2 = generation_allocation_start (youngest_generation);
23701 alloc_allocated = start2 + Align (size (start2));
23705 finish = GetCycleCount32();
23706 sweep_time = finish - start;
23710 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23712 assert ((tree != NULL));
23714 int right_node = node_right_child (tree);
23715 int left_node = node_left_child (tree);
23716 args->highest_plug = 0;
23719 if (! (0 == left_node))
23721 make_free_list_in_brick (tree + left_node, args);
23725 uint8_t* plug = tree;
23726 size_t gap_size = node_gap_size (tree);
23727 uint8_t* gap = (plug - gap_size);
23728 dprintf (3,("Making free list %Ix len %d in %d",
23729 //dprintf (3,("F: %Ix len %Ix in %d",
23730 (size_t)gap, gap_size, args->free_list_gen_number));
23731 args->highest_plug = tree;
23733 if (is_plug_padded (plug))
23735 dprintf (3, ("%Ix padded", plug));
23736 clear_plug_padded (plug);
23738 #endif //SHORT_PLUGS
23741 if ((args->current_gen_limit == MAX_PTR) ||
23742 ((plug >= args->current_gen_limit) &&
23743 ephemeral_pointer_p (plug)))
23745 dprintf(3,(" Crossing Generation boundary at %Ix",
23746 (size_t)args->current_gen_limit));
23747 if (!(args->current_gen_limit == MAX_PTR))
23749 args->free_list_gen_number--;
23750 args->free_list_gen = generation_of (args->free_list_gen_number);
23752 dprintf(3,( " Fixing generation start of %d to: %Ix",
23753 args->free_list_gen_number, (size_t)gap));
23755 reset_allocation_pointers (args->free_list_gen, gap);
23756 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23758 if ((gap_size >= (2*Align (min_obj_size))))
23760 dprintf(3,(" Splitting the gap in two %Id left",
23762 make_unused_array (gap, Align(min_obj_size));
23763 gap_size = (gap_size - Align(min_obj_size));
23764 gap = (gap + Align(min_obj_size));
23768 make_unused_array (gap, gap_size);
23775 thread_gap (gap, gap_size, args->free_list_gen);
23776 add_gen_free (args->free_list_gen->gen_num, gap_size);
23778 if (! (0 == right_node))
23780 make_free_list_in_brick (tree + right_node, args);
23786 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23788 assert (generation_allocation_start (gen));
23791 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23793 gen0_big_free_spaces += size;
23796 assert ((heap_segment_rw (generation_start_segment (gen))!=
23797 ephemeral_heap_segment) ||
23798 (gap_start > generation_allocation_start (gen)));
23799 // The beginning of a segment gap is not aligned
23800 assert (size >= Align (min_obj_size));
23801 make_unused_array (gap_start, size,
23802 (!settings.concurrent && (gen != youngest_generation)),
23803 (gen->gen_num == max_generation));
23804 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23806 if ((size >= min_free_list))
23808 generation_free_list_space (gen) += size;
23809 generation_allocator (gen)->thread_item (gap_start, size);
23813 generation_free_obj_space (gen) += size;
23818 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23820 assert (generation_allocation_start (gen));
23821 if (size >= min_free_list)
23823 generation_free_list_space (gen) += size;
23824 generation_allocator (gen)->thread_item_front (gap_start, size);
23828 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23830 dprintf (3, ("Making unused array [%Ix, %Ix[",
23831 (size_t)x, (size_t)(x+size)));
23832 assert (size >= Align (min_obj_size));
23834 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23835 // check_batch_mark_array_bits (x, x+size);
23836 //#endif //VERIFY_HEAP && BACKGROUND_GC
23839 reset_memory (x, size);
23841 ((CObjectHeader*)x)->SetFree(size);
23846 #error "This won't work on big endian platforms"
23849 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23851 if (size_as_object < size)
23854 // If the size is more than 4GB, we need to create multiple objects because of
23855 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23856 // size is ignored in regular object size computation.
23858 uint8_t * tmp = x + size_as_object;
23859 size_t remaining_size = size - size_as_object;
23861 while (remaining_size > UINT32_MAX)
23863 // Make sure that there will be at least Align(min_obj_size) left
23864 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23865 - Align (min_obj_size, get_alignment_constant (FALSE));
23867 ((CObjectHeader*)tmp)->SetFree(current_size);
23869 remaining_size -= current_size;
23870 tmp += current_size;
23873 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23878 clear_card_for_addresses (x, x + Align(size));
23881 // Clear memory set by make_unused_array.
23882 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23884 // Also clear the sync block
23885 *(((PTR_PTR)x)-1) = 0;
23887 ((CObjectHeader*)x)->UnsetFree();
23892 #error "This won't work on big endian platforms"
23895 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23896 // from make_unused_array since we cannot depend on the object sizes in memory.
23897 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23899 if (size_as_object < size)
23901 uint8_t * tmp = x + size_as_object;
23902 size_t remaining_size = size - size_as_object;
23904 while (remaining_size > UINT32_MAX)
23906 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23907 - Align (min_obj_size, get_alignment_constant (FALSE));
23909 ((CObjectHeader*)tmp)->UnsetFree();
23911 remaining_size -= current_size;
23912 tmp += current_size;
23915 ((CObjectHeader*)tmp)->UnsetFree();
23918 UNREFERENCED_PARAMETER(size);
23923 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23925 uint8_t* candidate = 0;
23929 if (tree < old_address)
23931 if ((cn = node_right_child (tree)) != 0)
23933 assert (candidate < tree);
23936 Prefetch (tree - 8);
23942 else if (tree > old_address)
23944 if ((cn = node_left_child (tree)) != 0)
23947 Prefetch (tree - 8);
23955 if (tree <= old_address)
23957 else if (candidate)
23963 #ifdef FEATURE_BASICFREEZE
23964 bool gc_heap::frozen_object_p (Object* obj)
23966 #ifdef MULTIPLE_HEAPS
23967 #ifdef SEG_MAPPING_TABLE
23968 heap_segment* pSegment = seg_mapping_table_segment_of((uint8_t*)obj);
23970 ptrdiff_t delta = 0;
23971 heap_segment* pSegment = segment_of ((uint8_t*)obj, delta);
23973 #else //MULTIPLE_HEAPS
23974 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23975 _ASSERTE(pSegment);
23976 #endif //MULTIPLE_HEAPS
23978 return heap_segment_read_only_p(pSegment);
23980 #endif // FEATURE_BASICFREEZE
23982 #ifdef FEATURE_REDHAWK
23983 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23984 // thing to do for other versions of the CLR.
23986 #endif // FEATURE_REDHAWK
23987 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23989 uint8_t* old_address = *pold_address;
23990 if (!((old_address >= gc_low) && (old_address < gc_high)))
23991 #ifdef MULTIPLE_HEAPS
23993 UNREFERENCED_PARAMETER(thread);
23994 if (old_address == 0)
23996 gc_heap* hp = heap_of (old_address);
23997 if ((hp == this) ||
23998 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
24001 #else //MULTIPLE_HEAPS
24003 #endif //MULTIPLE_HEAPS
24004 // delta translates old_address into address_gc (old_address);
24005 size_t brick = brick_of (old_address);
24006 int brick_entry = brick_table [ brick ];
24007 uint8_t* new_address = old_address;
24008 if (! ((brick_entry == 0)))
24012 while (brick_entry < 0)
24014 brick = (brick + brick_entry);
24015 brick_entry = brick_table [ brick ];
24017 uint8_t* old_loc = old_address;
24019 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
24021 if ((node <= old_loc))
24022 new_address = (old_address + node_relocation_distance (node));
24025 if (node_left_p (node))
24027 dprintf(3,(" L: %Ix", (size_t)node));
24028 new_address = (old_address +
24029 (node_relocation_distance (node) +
24030 node_gap_size (node)));
24035 brick_entry = brick_table [ brick ];
24041 *pold_address = new_address;
24045 #ifdef FEATURE_LOH_COMPACTION
24046 if (loh_compacted_p
24047 #ifdef FEATURE_BASICFREEZE
24048 && !frozen_object_p((Object*)old_address)
24049 #endif // FEATURE_BASICFREEZE
24052 *pold_address = old_address + loh_node_relocation_distance (old_address);
24055 #endif //FEATURE_LOH_COMPACTION
24057 *pold_address = new_address;
24062 gc_heap::check_class_object_demotion (uint8_t* obj)
24064 #ifdef COLLECTIBLE_CLASS
24065 if (is_collectible(obj))
24067 check_class_object_demotion_internal (obj);
24070 UNREFERENCED_PARAMETER(obj);
24071 #endif //COLLECTIBLE_CLASS
24074 #ifdef COLLECTIBLE_CLASS
24076 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
24078 if (settings.demotion)
24080 #ifdef MULTIPLE_HEAPS
24081 // We set the card without checking the demotion range 'cause at this point
24082 // the handle that points to the loader allocator object may or may not have
24083 // been relocated by other GC threads.
24084 set_card (card_of (obj));
24087 uint8_t* class_obj = get_class_object (obj);
24088 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
24089 uint8_t* temp_class_obj = class_obj;
24090 uint8_t** temp = &temp_class_obj;
24091 relocate_address (temp THREAD_NUMBER_ARG);
24093 check_demotion_helper (temp, obj);
24094 #endif //MULTIPLE_HEAPS
24098 #endif //COLLECTIBLE_CLASS
24101 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
24103 // detect if we are demoting an object
24104 if ((*pval < demotion_high) &&
24105 (*pval >= demotion_low))
24107 dprintf(3, ("setting card %Ix:%Ix",
24108 card_of((uint8_t*)pval),
24111 set_card (card_of (parent_obj));
24113 #ifdef MULTIPLE_HEAPS
24114 else if (settings.demotion)
24116 dprintf (4, ("Demotion active, computing heap_of object"));
24117 gc_heap* hp = heap_of (*pval);
24118 if ((*pval < hp->demotion_high) &&
24119 (*pval >= hp->demotion_low))
24121 dprintf(3, ("setting card %Ix:%Ix",
24122 card_of((uint8_t*)pval),
24125 set_card (card_of (parent_obj));
24128 #endif //MULTIPLE_HEAPS
24132 gc_heap::reloc_survivor_helper (uint8_t** pval)
24135 relocate_address (pval THREAD_NUMBER_ARG);
24137 check_demotion_helper (pval, (uint8_t*)pval);
24141 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
24144 if (contain_pointers (x))
24146 dprintf (3, ("$%Ix$", (size_t)x));
24148 go_through_object_nostart (method_table(x), x, s, pval,
24150 uint8_t* child = *pval;
24151 reloc_survivor_helper (pval);
24154 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
24159 check_class_object_demotion (x);
24163 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
24167 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
24168 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
24169 if (address_to_reloc)
24171 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
24174 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
24175 uint8_t* relocated_addr = *address_to_reloc;
24176 if ((relocated_addr < demotion_high) &&
24177 (relocated_addr >= demotion_low))
24179 dprintf (3, ("set card for location %Ix(%Ix)",
24180 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24182 set_card (card_of ((uint8_t*)address_to_set_card));
24184 #ifdef MULTIPLE_HEAPS
24185 else if (settings.demotion)
24187 gc_heap* hp = heap_of (relocated_addr);
24188 if ((relocated_addr < hp->demotion_high) &&
24189 (relocated_addr >= hp->demotion_low))
24191 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
24192 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24194 set_card (card_of ((uint8_t*)address_to_set_card));
24197 #endif //MULTIPLE_HEAPS
24200 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
24203 uint8_t* plug = pinned_plug (pinned_plug_entry);
24204 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
24205 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
24206 // address. Consider this scenario:
24207 // gen1 start | 3-ptr sized NP | PP
24209 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
24210 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
24211 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
24212 pre_plug_start += sizeof (uint8_t*);
24213 uint8_t** old_address = &pre_plug_start;
24215 uint8_t* old_val = (old_address ? *old_address : 0);
24216 relocate_address (old_address THREAD_NUMBER_ARG);
24219 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
24220 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
24223 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
24227 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
24230 uint8_t* plug = pinned_plug (pinned_plug_entry);
24234 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
24235 //if ((x + s) < plug)
24237 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
24238 // x, (x + s), (plug- (x + s)), plug));
24239 // GCToOSInterface::DebugBreak();
24242 relocate_pre_plug_info (pinned_plug_entry);
24245 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
24247 uint8_t* saved_plug_info_start = 0;
24248 uint8_t** saved_info_to_relocate = 0;
24252 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
24253 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24257 saved_plug_info_start = (plug - sizeof (plug_and_gap));
24258 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24261 uint8_t** current_saved_info_to_relocate = 0;
24262 uint8_t* child = 0;
24264 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
24266 if (contain_pointers (x))
24268 dprintf (3,("$%Ix$", (size_t)x));
24270 go_through_object_nostart (method_table(x), x, s, pval,
24272 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
24274 if ((uint8_t*)pval >= end)
24276 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
24277 child = *current_saved_info_to_relocate;
24278 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
24279 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
24280 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
24284 reloc_survivor_helper (pval);
24289 check_class_object_demotion (x);
24292 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
24295 while (x < plug_end)
24297 size_t s = size (x);
24298 uint8_t* next_obj = x + Align (s);
24299 Prefetch (next_obj);
24300 relocate_obj_helper (x, s);
24306 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
24307 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
24309 #if defined (_DEBUG) && defined (VERIFY_HEAP)
24310 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
24312 if (!verify_pinned_queue_p)
24315 if (settings.heap_expansion)
24318 for (size_t i = 0; i < mark_stack_tos; i++)
24320 mark& m = mark_stack_array[i];
24322 mark* pinned_plug_entry = pinned_plug_of(i);
24324 if (pinned_plug_entry->has_post_plug_info() &&
24325 pinned_plug_entry->post_short_p() &&
24326 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
24328 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
24329 // object after pin
24330 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
24331 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
24332 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
24334 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
24336 if (node_gap_size (next_obj) != *post_plug_debug)
24338 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
24339 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
24343 // can't do node_relocation_distance here as it clears the left bit.
24344 //if (node_relocation_distance (next_obj) != *post_plug_debug)
24345 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
24347 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
24348 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
24351 if (node_left_child (next_obj) > 0)
24353 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
24359 dprintf (3, ("%s verified", msg));
24361 #else // _DEBUG && VERIFY_HEAP
24362 UNREFERENCED_PARAMETER(msg);
24363 #endif // _DEBUG && VERIFY_HEAP
24366 #ifdef COLLECTIBLE_CLASS
24367 // We don't want to burn another ptr size space for pinned plugs to record this so just
24368 // set the card unconditionally for collectible objects if we are demoting.
24370 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
24372 if (settings.demotion)
24374 set_card (card_of (obj));
24377 #endif //COLLECTIBLE_CLASS
24379 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
24382 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
24383 BOOL is_pinned = (plug == p_plug);
24384 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
24386 plug_end += sizeof (gap_reloc_pair);
24388 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
24389 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
24391 verify_pins_with_post_plug_info("begin reloc short surv");
24393 while (x < plug_end)
24395 if (check_short_obj_p && ((plug_end - x) < (DWORD)min_pre_pin_obj_size))
24397 dprintf (3, ("last obj %Ix is short", x));
24401 #ifdef COLLECTIBLE_CLASS
24402 if (pinned_plug_entry->post_short_collectible_p())
24403 unconditional_set_card_collectible (x);
24404 #endif //COLLECTIBLE_CLASS
24406 // Relocate the saved references based on bits set.
24407 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
24408 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24409 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24411 if (pinned_plug_entry->post_short_bit_p (i))
24413 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24419 #ifdef COLLECTIBLE_CLASS
24420 if (pinned_plug_entry->pre_short_collectible_p())
24421 unconditional_set_card_collectible (x);
24422 #endif //COLLECTIBLE_CLASS
24424 relocate_pre_plug_info (pinned_plug_entry);
24426 // Relocate the saved references based on bits set.
24427 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
24428 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24429 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24431 if (pinned_plug_entry->pre_short_bit_p (i))
24433 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24441 size_t s = size (x);
24442 uint8_t* next_obj = x + Align (s);
24443 Prefetch (next_obj);
24445 if (next_obj >= plug_end)
24447 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
24448 next_obj, plug, plug_end));
24450 verify_pins_with_post_plug_info("before reloc short obj");
24452 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
24456 relocate_obj_helper (x, s);
24463 verify_pins_with_post_plug_info("end reloc short surv");
24466 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
24467 BOOL check_last_object_p,
24468 mark* pinned_plug_entry)
24470 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24471 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24473 if (check_last_object_p)
24475 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
24479 relocate_survivor_helper (plug, plug_end);
24483 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
24485 assert ((tree != NULL));
24487 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
24488 tree, args->last_plug,
24489 (tree + node_left_child (tree)),
24490 (tree + node_right_child (tree)),
24491 node_gap_size (tree)));
24493 if (node_left_child (tree))
24495 relocate_survivors_in_brick (tree + node_left_child (tree), args);
24498 uint8_t* plug = tree;
24499 BOOL has_post_plug_info_p = FALSE;
24500 BOOL has_pre_plug_info_p = FALSE;
24502 if (tree == oldest_pinned_plug)
24504 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24505 &has_post_plug_info_p);
24506 assert (tree == pinned_plug (args->pinned_plug_entry));
24508 dprintf (3, ("tree is the oldest pin: %Ix", tree));
24510 if (args->last_plug)
24512 size_t gap_size = node_gap_size (tree);
24513 uint8_t* gap = (plug - gap_size);
24514 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
24515 assert (gap_size >= Align (min_obj_size));
24516 uint8_t* last_plug_end = gap;
24518 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24521 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
24526 assert (!has_pre_plug_info_p);
24529 args->last_plug = plug;
24530 args->is_shortened = has_post_plug_info_p;
24531 if (has_post_plug_info_p)
24533 dprintf (3, ("setting %Ix as shortened", plug));
24535 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
24537 if (node_right_child (tree))
24539 relocate_survivors_in_brick (tree + node_right_child (tree), args);
24544 void gc_heap::update_oldest_pinned_plug()
24546 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
24549 void gc_heap::relocate_survivors (int condemned_gen_number,
24550 uint8_t* first_condemned_address)
24552 generation* condemned_gen = generation_of (condemned_gen_number);
24553 uint8_t* start_address = first_condemned_address;
24554 size_t current_brick = brick_of (start_address);
24555 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24557 PREFIX_ASSUME(current_heap_segment != NULL);
24559 uint8_t* end_address = 0;
24561 reset_pinned_queue_bos();
24562 update_oldest_pinned_plug();
24564 end_address = heap_segment_allocated (current_heap_segment);
24566 size_t end_brick = brick_of (end_address - 1);
24567 relocate_args args;
24569 args.high = gc_high;
24570 args.is_shortened = FALSE;
24571 args.pinned_plug_entry = 0;
24572 args.last_plug = 0;
24575 if (current_brick > end_brick)
24577 if (args.last_plug)
24580 assert (!(args.is_shortened));
24581 relocate_survivors_in_plug (args.last_plug,
24582 heap_segment_allocated (current_heap_segment),
24584 args.pinned_plug_entry);
24587 args.last_plug = 0;
24590 if (heap_segment_next_rw (current_heap_segment))
24592 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24593 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24594 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24603 int brick_entry = brick_table [ current_brick ];
24605 if (brick_entry >= 0)
24607 relocate_survivors_in_brick (brick_address (current_brick) +
24616 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24618 if (check_last_object_p)
24620 size += sizeof (gap_reloc_pair);
24621 mark* entry = args->pinned_plug_entry;
24623 if (args->is_shortened)
24625 assert (entry->has_post_plug_info());
24626 entry->swap_post_plug_and_saved_for_profiler();
24630 assert (entry->has_pre_plug_info());
24631 entry->swap_pre_plug_and_saved_for_profiler();
24635 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24636 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24637 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24639 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24641 if (check_last_object_p)
24643 mark* entry = args->pinned_plug_entry;
24645 if (args->is_shortened)
24647 entry->swap_post_plug_and_saved_for_profiler();
24651 entry->swap_pre_plug_and_saved_for_profiler();
24656 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24658 assert ((tree != NULL));
24659 if (node_left_child (tree))
24661 walk_relocation_in_brick (tree + node_left_child (tree), args);
24664 uint8_t* plug = tree;
24665 BOOL has_pre_plug_info_p = FALSE;
24666 BOOL has_post_plug_info_p = FALSE;
24668 if (tree == oldest_pinned_plug)
24670 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24671 &has_post_plug_info_p);
24672 assert (tree == pinned_plug (args->pinned_plug_entry));
24675 if (args->last_plug != 0)
24677 size_t gap_size = node_gap_size (tree);
24678 uint8_t* gap = (plug - gap_size);
24679 uint8_t* last_plug_end = gap;
24680 size_t last_plug_size = (last_plug_end - args->last_plug);
24681 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24682 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24684 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24685 if (!check_last_object_p)
24687 assert (last_plug_size >= Align (min_obj_size));
24690 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24694 assert (!has_pre_plug_info_p);
24697 dprintf (3, ("set args last plug to plug: %Ix", plug));
24698 args->last_plug = plug;
24699 args->is_shortened = has_post_plug_info_p;
24701 if (node_right_child (tree))
24703 walk_relocation_in_brick (tree + node_right_child (tree), args);
24707 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24709 generation* condemned_gen = generation_of (settings.condemned_generation);
24710 uint8_t* start_address = generation_allocation_start (condemned_gen);
24711 size_t current_brick = brick_of (start_address);
24712 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24714 PREFIX_ASSUME(current_heap_segment != NULL);
24716 reset_pinned_queue_bos();
24717 update_oldest_pinned_plug();
24718 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24719 walk_relocate_args args;
24720 args.is_shortened = FALSE;
24721 args.pinned_plug_entry = 0;
24722 args.last_plug = 0;
24723 args.profiling_context = profiling_context;
24728 if (current_brick > end_brick)
24730 if (args.last_plug)
24732 walk_plug (args.last_plug,
24733 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24736 args.last_plug = 0;
24738 if (heap_segment_next_rw (current_heap_segment))
24740 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24741 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24742 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24751 int brick_entry = brick_table [ current_brick ];
24752 if (brick_entry >= 0)
24754 walk_relocation_in_brick (brick_address (current_brick) +
24763 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24765 if (type == walk_for_gc)
24766 walk_survivors_relocation (context, fn);
24767 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24768 else if (type == walk_for_bgc)
24769 walk_survivors_for_bgc (context, fn);
24770 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24771 else if (type == walk_for_loh)
24772 walk_survivors_for_loh (context, fn);
24774 assert (!"unknown type!");
24777 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24778 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24780 // This should only be called for BGCs
24781 assert(settings.concurrent);
24783 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24785 BOOL small_object_segments = TRUE;
24786 int align_const = get_alignment_constant (small_object_segments);
24792 if (small_object_segments)
24794 //switch to large segment
24795 small_object_segments = FALSE;
24797 align_const = get_alignment_constant (small_object_segments);
24798 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24800 PREFIX_ASSUME(seg != NULL);
24808 uint8_t* o = heap_segment_mem (seg);
24809 uint8_t* end = heap_segment_allocated (seg);
24813 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24815 o += Align (size (o), align_const);
24819 // It's survived. Make a fake plug, starting at o,
24820 // and send the event
24822 uint8_t* plug_start = o;
24824 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24826 o += Align (size (o), align_const);
24833 uint8_t* plug_end = o;
24837 0, // Reloc distance == 0 as this is non-compacting
24839 false, // Non-compacting
24843 seg = heap_segment_next (seg);
24846 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24848 void gc_heap::relocate_phase (int condemned_gen_number,
24849 uint8_t* first_condemned_address)
24852 sc.thread_number = heap_number;
24853 sc.promotion = FALSE;
24854 sc.concurrent = FALSE;
24860 start = GetCycleCount32();
24863 // %type% category = quote (relocate);
24864 dprintf (2,("---- Relocate phase -----"));
24866 #ifdef MULTIPLE_HEAPS
24867 //join all threads to make sure they are synchronized
24868 dprintf(3, ("Joining after end of plan"));
24869 gc_t_join.join(this, gc_join_begin_relocate_phase);
24870 if (gc_t_join.joined())
24871 #endif //MULTIPLE_HEAPS
24874 #ifdef MULTIPLE_HEAPS
24876 //join all threads to make sure they are synchronized
24877 dprintf(3, ("Restarting for relocation"));
24878 gc_t_join.restart();
24879 #endif //MULTIPLE_HEAPS
24882 dprintf(3,("Relocating roots"));
24883 GCScan::GcScanRoots(GCHeap::Relocate,
24884 condemned_gen_number, max_generation, &sc);
24886 verify_pins_with_post_plug_info("after reloc stack");
24888 #ifdef BACKGROUND_GC
24889 if (recursive_gc_sync::background_running_p())
24891 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24893 #endif //BACKGROUND_GC
24895 if (condemned_gen_number != max_generation)
24897 dprintf(3,("Relocating cross generation pointers"));
24898 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24899 verify_pins_with_post_plug_info("after reloc cards");
24901 if (condemned_gen_number != max_generation)
24903 dprintf(3,("Relocating cross generation pointers for large objects"));
24904 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24908 #ifdef FEATURE_LOH_COMPACTION
24909 if (loh_compacted_p)
24911 assert (settings.condemned_generation == max_generation);
24912 relocate_in_loh_compact();
24915 #endif //FEATURE_LOH_COMPACTION
24917 relocate_in_large_objects ();
24921 dprintf(3,("Relocating survivors"));
24922 relocate_survivors (condemned_gen_number,
24923 first_condemned_address);
24926 #ifdef FEATURE_PREMORTEM_FINALIZATION
24927 dprintf(3,("Relocating finalization data"));
24928 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24930 #endif // FEATURE_PREMORTEM_FINALIZATION
24935 dprintf(3,("Relocating handle table"));
24936 GCScan::GcScanHandles(GCHeap::Relocate,
24937 condemned_gen_number, max_generation, &sc);
24940 #ifdef MULTIPLE_HEAPS
24941 //join all threads to make sure they are synchronized
24942 dprintf(3, ("Joining after end of relocation"));
24943 gc_t_join.join(this, gc_join_relocate_phase_done);
24945 #endif //MULTIPLE_HEAPS
24948 finish = GetCycleCount32();
24949 reloc_time = finish - start;
24952 dprintf(2,( "---- End of Relocate phase ----"));
24955 // This compares to see if tree is the current pinned plug and returns info
24956 // for this pinned plug. Also advances the pinned queue if that's the case.
24958 // We don't change the values of the plug info if tree is not the same as
24959 // the current pinned plug - the caller is responsible for setting the right
24960 // values to begin with.
24962 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24963 // where it passes FALSE to deque_p, change it to use the same optimization
24964 // as relocate. Not as essential since realloc is already a slow path.
24965 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24966 BOOL* has_pre_plug_info_p,
24967 BOOL* has_post_plug_info_p,
24970 if (!pinned_plug_que_empty_p())
24972 mark* oldest_entry = oldest_pin();
24973 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24974 if (tree == oldest_plug)
24976 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24977 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24981 deque_pinned_plug();
24984 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24986 (*has_pre_plug_info_p ? 1 : 0),
24987 (*has_post_plug_info_p ? 1 : 0)));
24989 return oldest_entry;
24996 // This also deques the oldest entry and update the oldest plug
24997 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24998 BOOL* has_post_plug_info_p)
25000 mark* oldest_entry = oldest_pin();
25001 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
25002 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
25004 deque_pinned_plug();
25005 update_oldest_pinned_plug();
25006 return oldest_entry;
25010 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25013 copy_cards_for_addresses (dest, src, len);
25015 clear_card_for_addresses (dest, dest + len);
25018 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
25019 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
25020 // we won't need to individually recover each overwritten part of plugs.
25022 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25026 #ifdef BACKGROUND_GC
25027 if (current_c_gc_state == c_gc_state_marking)
25029 //TODO: should look to see whether we should consider changing this
25030 // to copy a consecutive region of the mark array instead.
25031 copy_mark_bits_for_addresses (dest, src, len);
25033 #endif //BACKGROUND_GC
25034 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25035 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25036 memcopy (dest - plug_skew, src - plug_skew, len);
25037 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25038 if (SoftwareWriteWatch::IsEnabledForGCHeap())
25040 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
25041 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
25042 // object at (src + len), so it can be ignored anyway.
25043 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
25045 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25046 copy_cards_range (dest, src, len, copy_cards_p);
25050 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
25053 uint8_t* reloc_plug = plug + args->last_plug_relocation;
25055 if (check_last_object_p)
25057 size += sizeof (gap_reloc_pair);
25058 mark* entry = args->pinned_plug_entry;
25060 if (args->is_shortened)
25062 assert (entry->has_post_plug_info());
25063 entry->swap_post_plug_and_saved();
25067 assert (entry->has_pre_plug_info());
25068 entry->swap_pre_plug_and_saved();
25072 int old_brick_entry = brick_table [brick_of (plug)];
25074 assert (node_relocation_distance (plug) == args->last_plug_relocation);
25076 #ifdef FEATURE_STRUCTALIGN
25077 ptrdiff_t alignpad = node_alignpad(plug);
25080 make_unused_array (reloc_plug - alignpad, alignpad);
25081 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
25083 // The alignment padding is straddling one or more bricks;
25084 // it has to be the last "object" of its first brick.
25085 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
25088 #else // FEATURE_STRUCTALIGN
25089 size_t unused_arr_size = 0;
25090 BOOL already_padded_p = FALSE;
25092 if (is_plug_padded (plug))
25094 already_padded_p = TRUE;
25095 clear_plug_padded (plug);
25096 unused_arr_size = Align (min_obj_size);
25098 #endif //SHORT_PLUGS
25099 if (node_realigned (plug))
25101 unused_arr_size += switch_alignment_size (already_padded_p);
25104 if (unused_arr_size != 0)
25106 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
25108 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
25110 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
25111 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
25112 // The alignment padding is straddling one or more bricks;
25113 // it has to be the last "object" of its first brick.
25114 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
25117 #endif // FEATURE_STRUCTALIGN
25120 if (is_plug_padded (plug))
25122 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
25124 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
25126 // The alignment padding is straddling one or more bricks;
25127 // it has to be the last "object" of its first brick.
25128 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
25131 #endif //SHORT_PLUGS
25133 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
25135 if (args->check_gennum_p)
25137 int src_gennum = args->src_gennum;
25138 if (src_gennum == -1)
25140 src_gennum = object_gennum (plug);
25143 int dest_gennum = object_gennum_plan (reloc_plug);
25145 if (src_gennum < dest_gennum)
25147 generation_allocation_size (generation_of (dest_gennum)) += size;
25151 size_t current_reloc_brick = args->current_compacted_brick;
25153 if (brick_of (reloc_plug) != current_reloc_brick)
25155 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
25156 current_reloc_brick, brick_of (reloc_plug)));
25158 if (args->before_last_plug)
25160 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
25161 current_reloc_brick,
25162 args->before_last_plug,
25163 (args->before_last_plug - brick_address (current_reloc_brick))));
25166 set_brick (current_reloc_brick,
25167 args->before_last_plug - brick_address (current_reloc_brick));
25170 current_reloc_brick = brick_of (reloc_plug);
25172 size_t end_brick = brick_of (reloc_plug + size-1);
25173 if (end_brick != current_reloc_brick)
25175 // The plug is straddling one or more bricks
25176 // It has to be the last plug of its first brick
25177 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
25178 current_reloc_brick, (size_t)reloc_plug,
25179 (reloc_plug - brick_address (current_reloc_brick))));
25182 set_brick (current_reloc_brick,
25183 reloc_plug - brick_address (current_reloc_brick));
25185 // update all intervening brick
25186 size_t brick = current_reloc_brick + 1;
25187 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
25188 brick, (end_brick - 1)));
25189 while (brick < end_brick)
25191 set_brick (brick, -1);
25194 // code last brick offset as a plug address
25195 args->before_last_plug = brick_address (end_brick) -1;
25196 current_reloc_brick = end_brick;
25197 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
25198 args->before_last_plug, current_reloc_brick));
25202 dprintf (3, ("still in the same brick: %Ix", end_brick));
25203 args->before_last_plug = reloc_plug;
25205 args->current_compacted_brick = current_reloc_brick;
25207 if (check_last_object_p)
25209 mark* entry = args->pinned_plug_entry;
25211 if (args->is_shortened)
25213 entry->swap_post_plug_and_saved();
25217 entry->swap_pre_plug_and_saved();
25222 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
25224 assert (tree != NULL);
25225 int left_node = node_left_child (tree);
25226 int right_node = node_right_child (tree);
25227 ptrdiff_t relocation = node_relocation_distance (tree);
25233 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
25234 compact_in_brick ((tree + left_node), args);
25237 uint8_t* plug = tree;
25238 BOOL has_pre_plug_info_p = FALSE;
25239 BOOL has_post_plug_info_p = FALSE;
25241 if (tree == oldest_pinned_plug)
25243 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
25244 &has_post_plug_info_p);
25245 assert (tree == pinned_plug (args->pinned_plug_entry));
25248 if (args->last_plug != 0)
25250 size_t gap_size = node_gap_size (tree);
25251 uint8_t* gap = (plug - gap_size);
25252 uint8_t* last_plug_end = gap;
25253 size_t last_plug_size = (last_plug_end - args->last_plug);
25254 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
25255 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
25257 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
25258 if (!check_last_object_p)
25260 assert (last_plug_size >= Align (min_obj_size));
25263 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
25267 assert (!has_pre_plug_info_p);
25270 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
25271 args->last_plug = plug;
25272 args->last_plug_relocation = relocation;
25273 args->is_shortened = has_post_plug_info_p;
25277 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
25278 compact_in_brick ((tree + right_node), args);
25282 void gc_heap::recover_saved_pinned_info()
25284 reset_pinned_queue_bos();
25286 while (!(pinned_plug_que_empty_p()))
25288 mark* oldest_entry = oldest_pin();
25289 oldest_entry->recover_plug_info();
25290 #ifdef GC_CONFIG_DRIVEN
25291 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
25292 record_interesting_data_point (idp_pre_and_post_pin);
25293 else if (oldest_entry->has_pre_plug_info())
25294 record_interesting_data_point (idp_pre_pin);
25295 else if (oldest_entry->has_post_plug_info())
25296 record_interesting_data_point (idp_post_pin);
25297 #endif //GC_CONFIG_DRIVEN
25299 deque_pinned_plug();
25303 void gc_heap::compact_phase (int condemned_gen_number,
25304 uint8_t* first_condemned_address,
25307 // %type% category = quote (compact);
25311 start = GetCycleCount32();
25313 generation* condemned_gen = generation_of (condemned_gen_number);
25314 uint8_t* start_address = first_condemned_address;
25315 size_t current_brick = brick_of (start_address);
25316 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
25318 PREFIX_ASSUME(current_heap_segment != NULL);
25320 reset_pinned_queue_bos();
25321 update_oldest_pinned_plug();
25323 BOOL reused_seg = expand_reused_seg_p();
25326 for (int i = 1; i <= max_generation; i++)
25328 generation_allocation_size (generation_of (i)) = 0;
25332 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
25334 size_t end_brick = brick_of (end_address-1);
25336 args.last_plug = 0;
25337 args.before_last_plug = 0;
25338 args.current_compacted_brick = ~((size_t)1);
25339 args.is_shortened = FALSE;
25340 args.pinned_plug_entry = 0;
25341 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
25342 args.check_gennum_p = reused_seg;
25343 if (args.check_gennum_p)
25345 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25348 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
25349 first_condemned_address, brick_of (first_condemned_address)));
25351 #ifdef MULTIPLE_HEAPS
25353 if (gc_t_join.joined())
25355 #endif //MULTIPLE_HEAPS
25357 #ifdef MULTIPLE_HEAPS
25358 dprintf(3, ("Restarting for compaction"));
25359 gc_t_join.restart();
25361 #endif //MULTIPLE_HEAPS
25363 reset_pinned_queue_bos();
25365 #ifdef FEATURE_LOH_COMPACTION
25366 if (loh_compacted_p)
25370 #endif //FEATURE_LOH_COMPACTION
25372 if ((start_address < end_address) ||
25373 (condemned_gen_number == max_generation))
25377 if (current_brick > end_brick)
25379 if (args.last_plug != 0)
25381 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
25382 compact_plug (args.last_plug,
25383 (heap_segment_allocated (current_heap_segment) - args.last_plug),
25388 if (heap_segment_next_rw (current_heap_segment))
25390 current_heap_segment = heap_segment_next_rw (current_heap_segment);
25391 current_brick = brick_of (heap_segment_mem (current_heap_segment));
25392 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25393 args.last_plug = 0;
25394 if (args.check_gennum_p)
25396 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25402 if (args.before_last_plug !=0)
25404 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
25405 args.current_compacted_brick, (size_t)args.before_last_plug));
25406 assert (args.current_compacted_brick != ~1u);
25407 set_brick (args.current_compacted_brick,
25408 args.before_last_plug - brick_address (args.current_compacted_brick));
25414 int brick_entry = brick_table [ current_brick ];
25415 dprintf (3, ("B: %Ix(%Ix)->%Ix",
25416 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
25418 if (brick_entry >= 0)
25420 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
25429 recover_saved_pinned_info();
25432 finish = GetCycleCount32();
25433 compact_time = finish - start;
25436 concurrent_print_time_delta ("compact end");
25438 dprintf(2,("---- End of Compact phase ----"));
25441 #ifdef MULTIPLE_HEAPS
25444 #pragma warning(push)
25445 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25447 void gc_heap::gc_thread_stub (void* arg)
25449 gc_heap* heap = (gc_heap*)arg;
25450 if (!gc_thread_no_affinitize_p)
25452 GCThreadAffinity affinity;
25453 affinity.Group = GCThreadAffinity::None;
25454 affinity.Processor = GCThreadAffinity::None;
25456 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
25457 // CPU groups because the process mask, processor number, and group number are all
25458 // readily available.
25459 if (GCToOSInterface::CanEnableGCCPUGroups())
25460 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
25462 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
25464 if (!GCToOSInterface::SetThreadAffinity(&affinity))
25466 dprintf(1, ("Failed to set thread affinity for server GC thread"));
25470 // server GC threads run at a higher priority than normal.
25471 GCToOSInterface::BoostThreadPriority();
25472 _alloca (256*heap->heap_number);
25473 heap->gc_thread_function();
25476 #pragma warning(pop)
25479 #endif //MULTIPLE_HEAPS
25481 #ifdef BACKGROUND_GC
25484 #pragma warning(push)
25485 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25487 void gc_heap::bgc_thread_stub (void* arg)
25489 gc_heap* heap = (gc_heap*)arg;
25490 heap->bgc_thread = GCToEEInterface::GetThread();
25491 assert(heap->bgc_thread != nullptr);
25492 heap->bgc_thread_function();
25495 #pragma warning(pop)
25498 #endif //BACKGROUND_GC
25500 /*------------------ Background GC ----------------------------*/
25502 #ifdef BACKGROUND_GC
25504 void gc_heap::background_drain_mark_list (int thread)
25506 UNREFERENCED_PARAMETER(thread);
25508 size_t saved_c_mark_list_index = c_mark_list_index;
25510 if (saved_c_mark_list_index)
25512 concurrent_print_time_delta ("SML");
25514 while (c_mark_list_index != 0)
25516 size_t current_index = c_mark_list_index - 1;
25517 uint8_t* o = c_mark_list [current_index];
25518 background_mark_object (o THREAD_NUMBER_ARG);
25519 c_mark_list_index--;
25521 if (saved_c_mark_list_index)
25524 concurrent_print_time_delta ("EML");
25527 fire_drain_mark_list_event (saved_c_mark_list_index);
25531 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
25532 #ifdef MULTIPLE_HEAPS
25533 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
25534 // them. So we can use the same static variables.
25535 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25537 // Whenever we call this method there may have been preceding object promotions. So set
25538 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25539 // based on the how the scanning proceeded).
25540 s_fUnscannedPromotions = TRUE;
25542 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
25543 // the state of this thread's portion of the dependent handle table. That's because promotions on other
25544 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
25545 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
25546 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
25547 // as all the others or they'll get out of step).
25550 // The various worker threads are all currently racing in this code. We need to work out if at least
25551 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
25552 // dependent handle table when both of the following conditions apply:
25553 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
25554 // object happens to correspond to a primary in one of our handles we might potentially have to
25555 // promote the associated secondary).
25556 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
25558 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
25559 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
25560 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
25561 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
25562 // follows below. Note that we can't read this outside of the join since on any iteration apart from
25563 // the first threads will be racing between reading this value and completing their previous
25564 // iteration's table scan.
25566 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
25567 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
25568 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
25569 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
25570 // we're safely joined.
25571 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25572 s_fUnpromotedHandles = TRUE;
25574 // Synchronize all the threads so we can read our state variables safely. The following shared
25575 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
25576 // single thread inside the join.
25577 bgc_t_join.join(this, gc_join_scan_dependent_handles);
25578 if (bgc_t_join.joined())
25580 // We're synchronized so it's safe to read our shared state variables. We update another shared
25581 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25582 // the loop. We scan if there has been at least one object promotion since last time and at least
25583 // one thread has a dependent handle table with a potential handle promotion possible.
25584 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25586 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25587 // value for the next call if we're terminating the loop).
25588 s_fUnscannedPromotions = FALSE;
25589 s_fUnpromotedHandles = FALSE;
25591 if (!s_fScanRequired)
25593 uint8_t* all_heaps_max = 0;
25594 uint8_t* all_heaps_min = MAX_PTR;
25596 for (i = 0; i < n_heaps; i++)
25598 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25599 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25600 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25601 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25603 for (i = 0; i < n_heaps; i++)
25605 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25606 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25610 // Restart all the workers.
25611 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25612 bgc_t_join.restart();
25615 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25616 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25617 // global flag indicating that at least one object promotion may have occurred (the usual comment
25618 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25619 // exit the method since we unconditionally set this variable on method entry anyway).
25620 if (background_process_mark_overflow (sc->concurrent))
25621 s_fUnscannedPromotions = TRUE;
25623 // If we decided that no scan was required we can terminate the loop now.
25624 if (!s_fScanRequired)
25627 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25628 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25629 // could miss noting the promotion of some primary objects).
25630 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25631 if (bgc_t_join.joined())
25633 // Restart all the workers.
25634 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25635 bgc_t_join.restart();
25638 // If the portion of the dependent handle table managed by this worker has handles that could still be
25639 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25640 // could require a rescan of handles on this or other workers.
25641 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25642 if (GCScan::GcDhReScan(sc))
25643 s_fUnscannedPromotions = TRUE;
25647 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25649 // Whenever we call this method there may have been preceding object promotions. So set
25650 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25651 // based on the how the scanning proceeded).
25652 bool fUnscannedPromotions = true;
25654 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25655 // scan without performing any new promotions.
25656 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25658 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25659 fUnscannedPromotions = false;
25661 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25662 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25663 // additional objects now appear to be promoted and we should set the flag.
25664 if (background_process_mark_overflow (sc->concurrent))
25665 fUnscannedPromotions = true;
25667 // Perform the scan and set the flag if any promotions resulted.
25668 if (GCScan::GcDhReScan (sc))
25669 fUnscannedPromotions = true;
25672 // Perform a last processing of any overflowed mark stack.
25673 background_process_mark_overflow (sc->concurrent);
25675 #endif //MULTIPLE_HEAPS
25677 void gc_heap::recover_bgc_settings()
25679 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25681 dprintf (2, ("restoring bgc settings"));
25682 settings = saved_bgc_settings;
25683 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25687 void gc_heap::allow_fgc()
25689 assert (bgc_thread == GCToEEInterface::GetThread());
25690 bool bToggleGC = false;
25692 if (g_fSuspensionPending > 0)
25694 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25697 GCToEEInterface::DisablePreemptiveGC();
25702 BOOL gc_heap::should_commit_mark_array()
25704 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25707 void gc_heap::clear_commit_flag()
25709 generation* gen = generation_of (max_generation);
25710 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25715 if (gen != large_object_generation)
25717 gen = large_object_generation;
25718 seg = heap_segment_in_range (generation_start_segment (gen));
25726 if (seg->flags & heap_segment_flags_ma_committed)
25728 seg->flags &= ~heap_segment_flags_ma_committed;
25731 if (seg->flags & heap_segment_flags_ma_pcommitted)
25733 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25736 seg = heap_segment_next (seg);
25740 void gc_heap::clear_commit_flag_global()
25742 #ifdef MULTIPLE_HEAPS
25743 for (int i = 0; i < n_heaps; i++)
25745 g_heaps[i]->clear_commit_flag();
25748 clear_commit_flag();
25749 #endif //MULTIPLE_HEAPS
25752 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25755 size_t markw = mark_word_of (begin);
25756 size_t markw_end = mark_word_of (end);
25758 while (markw < markw_end)
25760 if (mark_array_addr[markw])
25762 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25763 markw, mark_array_addr[markw], mark_word_address (markw)));
25769 UNREFERENCED_PARAMETER(begin);
25770 UNREFERENCED_PARAMETER(end);
25771 UNREFERENCED_PARAMETER(mark_array_addr);
25775 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25777 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25780 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25782 uint32_t* new_card_table,
25783 uint8_t* new_lowest_address)
25785 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25787 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25788 uint8_t* end = heap_segment_reserved (seg);
25790 uint8_t* lowest = hp->background_saved_lowest_address;
25791 uint8_t* highest = hp->background_saved_highest_address;
25793 uint8_t* commit_start = NULL;
25794 uint8_t* commit_end = NULL;
25795 size_t commit_flag = 0;
25797 if ((highest >= start) &&
25800 if ((start >= lowest) && (end <= highest))
25802 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25803 start, end, lowest, highest));
25804 commit_flag = heap_segment_flags_ma_committed;
25808 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25809 start, end, lowest, highest));
25810 commit_flag = heap_segment_flags_ma_pcommitted;
25813 commit_start = max (lowest, start);
25814 commit_end = min (highest, end);
25816 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25821 if (new_card_table == 0)
25823 new_card_table = g_gc_card_table;
25826 if (hp->card_table != new_card_table)
25828 if (new_lowest_address == 0)
25830 new_lowest_address = g_gc_lowest_address;
25833 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25834 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25836 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25837 hp->card_table, new_card_table,
25838 hp->mark_array, ma));
25840 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25846 seg->flags |= commit_flag;
25852 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25854 size_t beg_word = mark_word_of (begin);
25855 size_t end_word = mark_word_of (align_on_mark_word (end));
25856 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25857 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25858 size_t size = (size_t)(commit_end - commit_start);
25860 #ifdef SIMPLE_DPRINTF
25861 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25863 beg_word, end_word,
25864 (end_word - beg_word) * sizeof (uint32_t),
25865 &mark_array_addr[beg_word],
25866 &mark_array_addr[end_word],
25867 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25868 commit_start, commit_end,
25870 #endif //SIMPLE_DPRINTF
25872 if (virtual_commit (commit_start, size))
25874 // We can only verify the mark array is cleared from begin to end, the first and the last
25875 // page aren't necessarily all cleared 'cause they could be used by other segments or
25877 verify_mark_array_cleared (begin, end, mark_array_addr);
25882 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25887 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25889 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25890 uint8_t* end = heap_segment_reserved (seg);
25892 #ifdef MULTIPLE_HEAPS
25893 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25894 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25896 uint8_t* lowest = background_saved_lowest_address;
25897 uint8_t* highest = background_saved_highest_address;
25898 #endif //MULTIPLE_HEAPS
25900 if ((highest >= start) &&
25903 start = max (lowest, start);
25904 end = min (highest, end);
25905 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25914 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25916 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25918 heap_segment_reserved (seg),
25920 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25922 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25925 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25927 UNREFERENCED_PARAMETER(mark_array_addr);
25929 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25930 lowest_address, highest_address, mark_array));
25932 generation* gen = generation_of (max_generation);
25933 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25938 if (gen != large_object_generation)
25940 gen = large_object_generation;
25941 seg = heap_segment_in_range (generation_start_segment (gen));
25949 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25951 if (!(seg->flags & heap_segment_flags_ma_committed))
25953 // For ro segments they could always be only partially in range so we'd
25954 // be calling this at the beginning of every BGC. We are not making this
25955 // more efficient right now - ro segments are currently only used by redhawk.
25956 if (heap_segment_read_only_p (seg))
25958 if ((heap_segment_mem (seg) >= lowest_address) &&
25959 (heap_segment_reserved (seg) <= highest_address))
25961 if (commit_mark_array_by_seg (seg, mark_array))
25963 seg->flags |= heap_segment_flags_ma_committed;
25972 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25973 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25974 if (commit_mark_array_by_range (start, end, mark_array))
25976 seg->flags |= heap_segment_flags_ma_pcommitted;
25986 // For normal segments they are by design completely in range so just
25987 // commit the whole mark array for each seg.
25988 if (commit_mark_array_by_seg (seg, mark_array))
25990 if (seg->flags & heap_segment_flags_ma_pcommitted)
25992 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25994 seg->flags |= heap_segment_flags_ma_committed;
26003 seg = heap_segment_next (seg);
26009 // This function doesn't check the commit flag since it's for a new array -
26010 // the mark_array flag for these segments will remain the same.
26011 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
26013 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
26014 generation* gen = generation_of (max_generation);
26015 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26020 if (gen != large_object_generation)
26022 gen = large_object_generation;
26023 seg = heap_segment_in_range (generation_start_segment (gen));
26031 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
26036 seg = heap_segment_next (seg);
26039 #ifdef MULTIPLE_HEAPS
26040 if (new_heap_segment)
26042 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
26047 #endif //MULTIPLE_HEAPS
26052 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
26054 #ifdef MULTIPLE_HEAPS
26055 for (int i = 0; i < n_heaps; i++)
26057 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
26063 if (!commit_new_mark_array (new_mark_array))
26067 #endif //MULTIPLE_HEAPS
26072 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
26074 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
26075 // been set to NULL.
26076 if (mark_array == NULL)
26081 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
26083 size_t flags = seg->flags;
26085 if ((flags & heap_segment_flags_ma_committed) ||
26086 (flags & heap_segment_flags_ma_pcommitted))
26088 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26089 uint8_t* end = heap_segment_reserved (seg);
26091 if (flags & heap_segment_flags_ma_pcommitted)
26093 start = max (lowest_address, start);
26094 end = min (highest_address, end);
26097 size_t beg_word = mark_word_of (start);
26098 size_t end_word = mark_word_of (align_on_mark_word (end));
26099 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
26100 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
26101 size_t size = (size_t)(decommit_end - decommit_start);
26103 #ifdef SIMPLE_DPRINTF
26104 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
26106 beg_word, end_word,
26107 (end_word - beg_word) * sizeof (uint32_t),
26108 &mark_array[beg_word],
26109 &mark_array[end_word],
26110 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
26111 decommit_start, decommit_end,
26113 #endif //SIMPLE_DPRINTF
26115 if (decommit_start < decommit_end)
26117 if (!virtual_decommit (decommit_start, size))
26119 dprintf (GC_TABLE_LOG, ("decommit on %Ix for %Id bytes failed",
26120 decommit_start, size));
26121 assert (!"decommit failed");
26125 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
26129 void gc_heap::background_mark_phase ()
26131 verify_mark_array_cleared();
26134 sc.thread_number = heap_number;
26135 sc.promotion = TRUE;
26136 sc.concurrent = FALSE;
26139 BOOL cooperative_mode = TRUE;
26140 #ifndef MULTIPLE_HEAPS
26141 const int thread = heap_number;
26142 #endif //!MULTIPLE_HEAPS
26144 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
26146 assert (settings.concurrent);
26151 start = GetCycleCount32();
26154 #ifdef FFIND_OBJECT
26155 if (gen0_must_clear_bricks > 0)
26156 gen0_must_clear_bricks--;
26157 #endif //FFIND_OBJECT
26159 background_soh_alloc_count = 0;
26160 background_loh_alloc_count = 0;
26161 bgc_overflow_count = 0;
26163 bpromoted_bytes (heap_number) = 0;
26164 static uint32_t num_sizedrefs = 0;
26166 background_min_overflow_address = MAX_PTR;
26167 background_max_overflow_address = 0;
26168 background_min_soh_overflow_address = MAX_PTR;
26169 background_max_soh_overflow_address = 0;
26170 processed_soh_overflow_p = FALSE;
26173 //set up the mark lists from g_mark_list
26174 assert (g_mark_list);
26175 mark_list = g_mark_list;
26176 //dont use the mark list for full gc
26177 //because multiple segments are more complex to handle and the list
26178 //is likely to overflow
26179 mark_list_end = &mark_list [0];
26180 mark_list_index = &mark_list [0];
26182 c_mark_list_index = 0;
26184 #ifndef MULTIPLE_HEAPS
26185 shigh = (uint8_t*) 0;
26187 #endif //MULTIPLE_HEAPS
26189 generation* gen = generation_of (max_generation);
26191 dprintf(3,("BGC: stack marking"));
26192 sc.concurrent = TRUE;
26194 GCScan::GcScanRoots(background_promote_callback,
26195 max_generation, max_generation,
26200 dprintf(3,("BGC: finalization marking"));
26201 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
26204 size_t total_loh_size = generation_size (max_generation + 1);
26205 bgc_begin_loh_size = total_loh_size;
26206 bgc_alloc_spin_loh = 0;
26207 bgc_loh_size_increased = 0;
26208 bgc_loh_allocated_in_free = 0;
26209 size_t total_soh_size = generation_sizes (generation_of (max_generation));
26211 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26214 //concurrent_print_time_delta ("copying stack roots");
26215 concurrent_print_time_delta ("CS");
26217 FIRE_EVENT(BGC1stNonConEnd);
26219 expanded_in_fgc = FALSE;
26220 saved_overflow_ephemeral_seg = 0;
26221 current_bgc_state = bgc_reset_ww;
26223 // we don't need a join here - just whichever thread that gets here
26224 // first can change the states and call restart_vm.
26225 // this is not true - we can't let the EE run when we are scanning stack.
26226 // since we now allow reset ww to run concurrently and have a join for it,
26227 // we can do restart ee on the 1st thread that got here. Make sure we handle the
26228 // sizedref handles correctly.
26229 #ifdef MULTIPLE_HEAPS
26230 bgc_t_join.join(this, gc_join_restart_ee);
26231 if (bgc_t_join.joined())
26232 #endif //MULTIPLE_HEAPS
26234 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26235 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
26236 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
26237 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
26239 concurrent_print_time_delta ("CRWW begin");
26241 #ifdef MULTIPLE_HEAPS
26242 for (int i = 0; i < n_heaps; i++)
26244 g_heaps[i]->reset_write_watch (FALSE);
26247 reset_write_watch (FALSE);
26248 #endif //MULTIPLE_HEAPS
26250 concurrent_print_time_delta ("CRWW");
26251 #endif //WRITE_WATCH
26252 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26254 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
26256 // this c_write is not really necessary because restart_vm
26257 // has an instruction that will flush the cpu cache (interlocked
26258 // or whatever) but we don't want to rely on that.
26259 dprintf (BGC_LOG, ("setting cm_in_progress"));
26260 c_write (cm_in_progress, TRUE);
26262 //restart all thread, doing the marking from the array
26263 assert (dont_restart_ee_p);
26264 dont_restart_ee_p = FALSE;
26267 GCToOSInterface::YieldThread (0);
26268 #ifdef MULTIPLE_HEAPS
26269 dprintf(3, ("Starting all gc threads for gc"));
26270 bgc_t_join.restart();
26271 #endif //MULTIPLE_HEAPS
26274 #ifdef MULTIPLE_HEAPS
26275 bgc_t_join.join(this, gc_join_after_reset);
26276 if (bgc_t_join.joined())
26277 #endif //MULTIPLE_HEAPS
26279 disable_preemptive (true);
26281 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26282 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
26283 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
26284 // pages during the concurrent reset.
26287 concurrent_print_time_delta ("CRWW begin");
26289 #ifdef MULTIPLE_HEAPS
26290 for (int i = 0; i < n_heaps; i++)
26292 g_heaps[i]->reset_write_watch (TRUE);
26295 reset_write_watch (TRUE);
26296 #endif //MULTIPLE_HEAPS
26298 concurrent_print_time_delta ("CRWW");
26299 #endif //WRITE_WATCH
26301 #ifdef MULTIPLE_HEAPS
26302 for (int i = 0; i < n_heaps; i++)
26304 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
26307 revisit_written_pages (TRUE, TRUE);
26308 #endif //MULTIPLE_HEAPS
26310 concurrent_print_time_delta ("CRW");
26311 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26313 #ifdef MULTIPLE_HEAPS
26314 for (int i = 0; i < n_heaps; i++)
26316 g_heaps[i]->current_bgc_state = bgc_mark_handles;
26319 current_bgc_state = bgc_mark_handles;
26320 #endif //MULTIPLE_HEAPS
26322 current_c_gc_state = c_gc_state_marking;
26324 enable_preemptive ();
26326 #ifdef MULTIPLE_HEAPS
26327 dprintf(3, ("Joining BGC threads after resetting writewatch"));
26328 bgc_t_join.restart();
26329 #endif //MULTIPLE_HEAPS
26332 disable_preemptive (true);
26334 if (num_sizedrefs > 0)
26336 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
26338 enable_preemptive ();
26340 #ifdef MULTIPLE_HEAPS
26341 bgc_t_join.join(this, gc_join_scan_sizedref_done);
26342 if (bgc_t_join.joined())
26344 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
26345 bgc_t_join.restart();
26347 #endif //MULTIPLE_HEAPS
26349 disable_preemptive (true);
26352 dprintf (3,("BGC: handle table marking"));
26353 GCScan::GcScanHandles(background_promote,
26354 max_generation, max_generation,
26356 //concurrent_print_time_delta ("concurrent marking handle table");
26357 concurrent_print_time_delta ("CRH");
26359 current_bgc_state = bgc_mark_stack;
26360 dprintf (2,("concurrent draining mark list"));
26361 background_drain_mark_list (thread);
26362 //concurrent_print_time_delta ("concurrent marking stack roots");
26363 concurrent_print_time_delta ("CRS");
26365 dprintf (2,("concurrent revisiting dirtied pages"));
26366 revisit_written_pages (TRUE);
26367 revisit_written_pages (TRUE);
26368 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
26369 concurrent_print_time_delta ("CRre");
26371 enable_preemptive ();
26373 #ifdef MULTIPLE_HEAPS
26374 bgc_t_join.join(this, gc_join_concurrent_overflow);
26375 if (bgc_t_join.joined())
26377 uint8_t* all_heaps_max = 0;
26378 uint8_t* all_heaps_min = MAX_PTR;
26380 for (i = 0; i < n_heaps; i++)
26382 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
26384 g_heaps[i]->background_max_overflow_address,
26385 g_heaps[i]->background_min_overflow_address));
26386 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
26387 all_heaps_max = g_heaps[i]->background_max_overflow_address;
26388 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
26389 all_heaps_min = g_heaps[i]->background_min_overflow_address;
26391 for (i = 0; i < n_heaps; i++)
26393 g_heaps[i]->background_max_overflow_address = all_heaps_max;
26394 g_heaps[i]->background_min_overflow_address = all_heaps_min;
26396 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
26397 bgc_t_join.restart();
26399 #endif //MULTIPLE_HEAPS
26401 disable_preemptive (true);
26403 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
26404 bgc_overflow_count = 0;
26405 background_process_mark_overflow (TRUE);
26406 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
26407 bgc_overflow_count = 0;
26408 //concurrent_print_time_delta ("concurrent processing mark overflow");
26409 concurrent_print_time_delta ("CRov");
26411 // Stop all threads, crawl all stacks and revisit changed pages.
26412 FIRE_EVENT(BGC1stConEnd);
26414 dprintf (2, ("Stopping the EE"));
26416 enable_preemptive ();
26418 #ifdef MULTIPLE_HEAPS
26419 bgc_t_join.join(this, gc_join_suspend_ee);
26420 if (bgc_t_join.joined())
26422 bgc_threads_sync_event.Reset();
26424 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
26425 bgc_t_join.restart();
26427 #endif //MULTIPLE_HEAPS
26429 if (heap_number == 0)
26431 enter_spin_lock (&gc_lock);
26435 bgc_threads_sync_event.Set();
26439 bgc_threads_sync_event.Wait(INFINITE, FALSE);
26440 dprintf (2, ("bgc_threads_sync_event is signalled"));
26443 assert (settings.concurrent);
26444 assert (settings.condemned_generation == max_generation);
26446 dprintf (2, ("clearing cm_in_progress"));
26447 c_write (cm_in_progress, FALSE);
26449 bgc_alloc_lock->check();
26451 current_bgc_state = bgc_final_marking;
26453 //concurrent_print_time_delta ("concurrent marking ended");
26454 concurrent_print_time_delta ("CR");
26456 FIRE_EVENT(BGC2ndNonConBegin);
26458 mark_absorb_new_alloc();
26460 // We need a join here 'cause find_object would complain if the gen0
26461 // bricks of another heap haven't been fixed up. So we need to make sure
26462 // that every heap's gen0 bricks are fixed up before we proceed.
26463 #ifdef MULTIPLE_HEAPS
26464 bgc_t_join.join(this, gc_join_after_absorb);
26465 if (bgc_t_join.joined())
26467 dprintf(3, ("Joining BGC threads after absorb"));
26468 bgc_t_join.restart();
26470 #endif //MULTIPLE_HEAPS
26472 // give VM a chance to do work
26473 GCToEEInterface::GcBeforeBGCSweepWork();
26475 //reset the flag, indicating that the EE no longer expect concurrent
26477 sc.concurrent = FALSE;
26479 total_loh_size = generation_size (max_generation + 1);
26480 total_soh_size = generation_sizes (generation_of (max_generation));
26482 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26484 dprintf (2, ("nonconcurrent marking stack roots"));
26485 GCScan::GcScanRoots(background_promote,
26486 max_generation, max_generation,
26488 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
26489 concurrent_print_time_delta ("NRS");
26491 // finalize_queue->EnterFinalizeLock();
26492 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
26493 // finalize_queue->LeaveFinalizeLock();
26495 dprintf (2, ("nonconcurrent marking handle table"));
26496 GCScan::GcScanHandles(background_promote,
26497 max_generation, max_generation,
26499 //concurrent_print_time_delta ("nonconcurrent marking handle table");
26500 concurrent_print_time_delta ("NRH");
26502 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
26503 revisit_written_pages (FALSE);
26504 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
26505 concurrent_print_time_delta ("NRre LOH");
26507 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26508 #ifdef MULTIPLE_HEAPS
26509 bgc_t_join.join(this, gc_join_disable_software_write_watch);
26510 if (bgc_t_join.joined())
26511 #endif // MULTIPLE_HEAPS
26513 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
26514 // avoid further perf penalty after the runtime is restarted
26515 SoftwareWriteWatch::DisableForGCHeap();
26517 #ifdef MULTIPLE_HEAPS
26518 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
26519 bgc_t_join.restart();
26520 #endif // MULTIPLE_HEAPS
26522 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26524 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
26525 bgc_overflow_count = 0;
26527 // Dependent handles need to be scanned with a special algorithm (see the header comment on
26528 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
26529 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
26530 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
26531 // The call to background_scan_dependent_handles is what will cycle through more iterations if
26532 // required and will also perform processing of any mark stack overflow once the dependent handle
26533 // table has been fully promoted.
26534 dprintf (2, ("1st dependent handle scan and process mark overflow"));
26535 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
26536 background_scan_dependent_handles (&sc);
26537 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
26538 concurrent_print_time_delta ("NR 1st Hov");
26540 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
26541 bgc_overflow_count = 0;
26543 #ifdef MULTIPLE_HEAPS
26544 bgc_t_join.join(this, gc_join_null_dead_short_weak);
26545 if (bgc_t_join.joined())
26546 #endif //MULTIPLE_HEAPS
26548 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
26550 #ifdef MULTIPLE_HEAPS
26551 dprintf(3, ("Joining BGC threads for short weak handle scan"));
26552 bgc_t_join.restart();
26553 #endif //MULTIPLE_HEAPS
26556 // null out the target of short weakref that were not promoted.
26557 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
26559 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
26560 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
26564 #ifdef MULTIPLE_HEAPS
26565 bgc_t_join.join(this, gc_join_scan_finalization);
26566 if (bgc_t_join.joined())
26568 dprintf(3, ("Joining BGC threads for finalization"));
26569 bgc_t_join.restart();
26571 #endif //MULTIPLE_HEAPS
26573 //Handle finalization.
26574 dprintf(3,("Marking finalization data"));
26575 //concurrent_print_time_delta ("bgc joined to mark finalization");
26576 concurrent_print_time_delta ("NRj");
26578 // finalize_queue->EnterFinalizeLock();
26579 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
26580 // finalize_queue->LeaveFinalizeLock();
26582 concurrent_print_time_delta ("NRF");
26585 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26586 bgc_overflow_count = 0;
26588 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26589 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26591 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26592 background_scan_dependent_handles (&sc);
26593 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26594 concurrent_print_time_delta ("NR 2nd Hov");
26596 #ifdef MULTIPLE_HEAPS
26597 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26598 if (bgc_t_join.joined())
26600 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26601 bgc_t_join.restart();
26603 #endif //MULTIPLE_HEAPS
26605 // null out the target of long weakref that were not promoted.
26606 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26607 concurrent_print_time_delta ("NR GcWeakPtrScan");
26609 #ifdef MULTIPLE_HEAPS
26610 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26611 if (bgc_t_join.joined())
26612 #endif //MULTIPLE_HEAPS
26614 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26615 // scan for deleted entries in the syncblk cache
26616 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26617 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26618 #ifdef MULTIPLE_HEAPS
26619 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26620 bgc_t_join.restart();
26621 #endif //MULTIPLE_HEAPS
26624 gen0_bricks_cleared = FALSE;
26626 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26627 generation_size (max_generation + 1),
26628 generation_sizes (generation_of (max_generation))));
26630 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26632 generation* gen = generation_of (gen_idx);
26633 dynamic_data* dd = dynamic_data_of (gen_idx);
26634 dd_begin_data_size (dd) = generation_size (gen_idx) -
26635 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26636 Align (size (generation_allocation_start (gen)));
26637 dd_survived_size (dd) = 0;
26638 dd_pinned_survived_size (dd) = 0;
26639 dd_artificial_pinned_survived_size (dd) = 0;
26640 dd_added_pinned_size (dd) = 0;
26643 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26644 PREFIX_ASSUME(seg != NULL);
26648 seg->flags &= ~heap_segment_flags_swept;
26650 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26652 // This can't happen...
26656 if (seg == ephemeral_heap_segment)
26658 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26662 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26665 dprintf (2, ("seg %Ix background allocated is %Ix",
26666 heap_segment_mem (seg),
26667 heap_segment_background_allocated (seg)));
26668 seg = heap_segment_next_rw (seg);
26671 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26672 // we can't let the user code consume the left over parts in these alloc contexts.
26673 repair_allocation_contexts (FALSE);
26676 finish = GetCycleCount32();
26677 mark_time = finish - start;
26680 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26681 generation_free_list_space (generation_of (max_generation)),
26682 generation_free_obj_space (generation_of (max_generation))));
26684 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26688 gc_heap::suspend_EE ()
26690 dprintf (2, ("suspend_EE"));
26691 #ifdef MULTIPLE_HEAPS
26692 gc_heap* hp = gc_heap::g_heaps[0];
26693 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26695 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26696 #endif //MULTIPLE_HEAPS
26699 #ifdef MULTIPLE_HEAPS
26701 gc_heap::bgc_suspend_EE ()
26703 for (int i = 0; i < n_heaps; i++)
26705 gc_heap::g_heaps[i]->reset_gc_done();
26708 dprintf (2, ("bgc_suspend_EE"));
26709 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26711 gc_started = FALSE;
26712 for (int i = 0; i < n_heaps; i++)
26714 gc_heap::g_heaps[i]->set_gc_done();
26719 gc_heap::bgc_suspend_EE ()
26723 dprintf (2, ("bgc_suspend_EE"));
26724 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26725 gc_started = FALSE;
26728 #endif //MULTIPLE_HEAPS
26731 gc_heap::restart_EE ()
26733 dprintf (2, ("restart_EE"));
26734 #ifdef MULTIPLE_HEAPS
26735 GCToEEInterface::RestartEE(FALSE);
26737 GCToEEInterface::RestartEE(FALSE);
26738 #endif //MULTIPLE_HEAPS
26741 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26745 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26746 generation_allocation_start (generation_of (max_generation-1)) :
26747 heap_segment_allocated (seg));
26748 return align_lower_page (end);
26752 return heap_segment_allocated (seg);
26756 void gc_heap::revisit_written_page (uint8_t* page,
26760 uint8_t*& last_page,
26761 uint8_t*& last_object,
26762 BOOL large_objects_p,
26763 size_t& num_marked_objects)
26765 UNREFERENCED_PARAMETER(seg);
26767 uint8_t* start_address = page;
26769 int align_const = get_alignment_constant (!large_objects_p);
26770 uint8_t* high_address = end;
26771 uint8_t* current_lowest_address = background_saved_lowest_address;
26772 uint8_t* current_highest_address = background_saved_highest_address;
26773 BOOL no_more_loop_p = FALSE;
26776 #ifndef MULTIPLE_HEAPS
26777 const int thread = heap_number;
26778 #endif //!MULTIPLE_HEAPS
26780 if (large_objects_p)
26786 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26787 || (start_address <= last_object))
26793 o = find_first_object (start_address, last_object);
26794 // We can visit the same object again, but on a different page.
26795 assert (o >= last_object);
26799 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26800 (size_t)page, (size_t)o,
26801 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26803 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26807 if (concurrent_p && large_objects_p)
26809 bgc_alloc_lock->bgc_mark_set (o);
26811 if (((CObjectHeader*)o)->IsFree())
26813 s = unused_array_size (o);
26825 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26827 assert (Align (s) >= Align (min_obj_size));
26829 uint8_t* next_o = o + Align (s, align_const);
26831 if (next_o >= start_address)
26833 #ifdef MULTIPLE_HEAPS
26836 // We set last_object here for SVR BGC here because SVR BGC has more than
26837 // one GC thread. When we have more than one GC thread we would run into this
26838 // situation if we skipped unmarked objects:
26839 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26841 // bgc thread 2 marks X and all its current children.
26842 // user thread comes along and dirties more (and later) pages in X.
26843 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26844 // on them because it had already skipped X. We need to detect that this object is now
26845 // marked and mark the children on the dirtied pages.
26846 // In the future if we have less BGC threads than we have heaps we should add
26847 // the check to the number of BGC threads.
26850 #endif //MULTIPLE_HEAPS
26852 if (contain_pointers (o) &&
26853 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26854 background_marked (o)))
26856 dprintf (3, ("going through %Ix", (size_t)o));
26857 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26858 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26860 no_more_loop_p = TRUE;
26863 uint8_t* oo = *poo;
26865 num_marked_objects++;
26866 background_mark_object (oo THREAD_NUMBER_ARG);
26871 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26873 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26874 ((CObjectHeader*)o)->IsFree() &&
26875 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26877 // We need to not skip the object here because of this corner scenario:
26878 // A large object was being allocated during BGC mark so we first made it
26879 // into a free object, then cleared its memory. In this loop we would detect
26880 // that it's a free object which normally we would skip. But by the next time
26881 // we call GetWriteWatch we could still be on this object and the object had
26882 // been made into a valid object and some of its memory was changed. We need
26883 // to be sure to process those written pages so we can't skip the object just
26886 // Similarly, when using software write watch, don't advance last_object when
26887 // the current object is a free object that spans beyond the current page or
26888 // high_address. Software write watch acquires gc_lock before the concurrent
26889 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26890 // happen at that point and allocate from this free region, so when
26891 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26893 no_more_loop_p = TRUE;
26898 if (concurrent_p && large_objects_p)
26900 bgc_alloc_lock->bgc_mark_done ();
26902 if (no_more_loop_p)
26909 #ifdef MULTIPLE_HEAPS
26912 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26915 #endif //MULTIPLE_HEAPS
26920 dprintf (3,("Last object: %Ix", (size_t)last_object));
26921 last_page = align_write_watch_lower_page (o);
26924 // When reset_only_p is TRUE, we should only reset pages that are in range
26925 // because we need to consider the segments or part of segments that were
26926 // allocated out of range all live.
26927 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26930 if (concurrent_p && !reset_only_p)
26932 current_bgc_state = bgc_revisit_soh;
26935 size_t total_dirtied_pages = 0;
26936 size_t total_marked_objects = 0;
26938 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26940 PREFIX_ASSUME(seg != NULL);
26942 bool reset_watch_state = !!concurrent_p;
26943 bool is_runtime_suspended = !concurrent_p;
26944 BOOL small_object_segments = TRUE;
26945 int align_const = get_alignment_constant (small_object_segments);
26951 if (small_object_segments)
26953 //switch to large segment
26954 if (concurrent_p && !reset_only_p)
26956 current_bgc_state = bgc_revisit_loh;
26961 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26962 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26963 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26964 total_dirtied_pages = 0;
26965 total_marked_objects = 0;
26968 small_object_segments = FALSE;
26969 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26971 dprintf (3, ("now revisiting large object segments"));
26972 align_const = get_alignment_constant (small_object_segments);
26973 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26975 PREFIX_ASSUME(seg != NULL);
26983 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26987 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26988 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26993 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26994 //we need to truncate to the base of the page because
26995 //some newly allocated could exist beyond heap_segment_allocated
26996 //and if we reset the last page write watch status,
26997 // they wouldn't be guaranteed to be visited -> gc hole.
26998 uintptr_t bcount = array_size;
26999 uint8_t* last_page = 0;
27000 uint8_t* last_object = heap_segment_mem (seg);
27001 uint8_t* high_address = 0;
27003 BOOL skip_seg_p = FALSE;
27007 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
27008 (heap_segment_reserved (seg) <= background_saved_highest_address))
27010 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
27011 heap_segment_mem (seg), heap_segment_reserved (seg)));
27018 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
27022 base_address = max (base_address, background_saved_lowest_address);
27023 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
27026 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
27027 heap_segment_mem (seg), heap_segment_reserved (seg)));
27034 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
27035 high_address = min (high_address, background_saved_highest_address);
27039 high_address = high_page (seg, concurrent_p);
27042 if ((base_address < high_address) &&
27043 (bcount >= array_size))
27045 ptrdiff_t region_size = high_address - base_address;
27046 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
27048 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27049 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
27050 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
27051 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
27053 if (!is_runtime_suspended)
27055 enter_spin_lock(&gc_lock);
27057 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27059 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
27060 (void**)background_written_addresses,
27061 &bcount, is_runtime_suspended);
27063 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27064 if (!is_runtime_suspended)
27066 leave_spin_lock(&gc_lock);
27068 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27072 total_dirtied_pages += bcount;
27074 dprintf (3, ("Found %d pages [%Ix, %Ix[",
27075 bcount, (size_t)base_address, (size_t)high_address));
27080 for (unsigned i = 0; i < bcount; i++)
27082 uint8_t* page = (uint8_t*)background_written_addresses[i];
27083 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
27084 (size_t)page, (size_t)high_address));
27085 if (page < high_address)
27087 //search for marked objects in the page
27088 revisit_written_page (page, high_address, concurrent_p,
27089 seg, last_page, last_object,
27090 !small_object_segments,
27091 total_marked_objects);
27095 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
27096 assert (!"page shouldn't have exceeded limit");
27101 if (bcount >= array_size){
27102 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
27103 bcount = array_size;
27113 seg = heap_segment_next_rw (seg);
27116 #endif //WRITE_WATCH
27119 void gc_heap::background_grow_c_mark_list()
27121 assert (c_mark_list_index >= c_mark_list_length);
27122 BOOL should_drain_p = FALSE;
27124 #ifndef MULTIPLE_HEAPS
27125 const int thread = heap_number;
27126 #endif //!MULTIPLE_HEAPS
27128 dprintf (2, ("stack copy buffer overflow"));
27129 uint8_t** new_c_mark_list = 0;
27132 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
27134 should_drain_p = TRUE;
27138 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
27139 if (new_c_mark_list == 0)
27141 should_drain_p = TRUE;
27145 if (should_drain_p)
27148 dprintf (2, ("No more memory for the stacks copy, draining.."));
27149 //drain the list by marking its elements
27150 background_drain_mark_list (thread);
27154 assert (new_c_mark_list);
27155 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
27156 c_mark_list_length = c_mark_list_length*2;
27157 delete c_mark_list;
27158 c_mark_list = new_c_mark_list;
27162 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
27165 UNREFERENCED_PARAMETER(sc);
27166 //in order to save space on the array, mark the object,
27167 //knowing that it will be visited later
27168 assert (settings.concurrent);
27170 THREAD_NUMBER_FROM_CONTEXT;
27171 #ifndef MULTIPLE_HEAPS
27172 const int thread = 0;
27173 #endif //!MULTIPLE_HEAPS
27175 uint8_t* o = (uint8_t*)*ppObject;
27182 gc_heap* hp = gc_heap::heap_of (o);
27184 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
27189 #ifdef INTERIOR_POINTERS
27190 if (flags & GC_CALL_INTERIOR)
27192 o = hp->find_object (o, hp->background_saved_lowest_address);
27196 #endif //INTERIOR_POINTERS
27198 #ifdef FEATURE_CONSERVATIVE_GC
27199 // For conservative GC, a value on stack may point to middle of a free object.
27200 // In this case, we don't need to promote the pointer.
27201 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
27205 #endif //FEATURE_CONSERVATIVE_GC
27208 ((CObjectHeader*)o)->Validate();
27211 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
27212 if (o && (size (o) > loh_size_threshold))
27214 dprintf (3, ("Brc %Ix", (size_t)o));
27217 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
27219 hpt->background_grow_c_mark_list();
27221 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
27222 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
27224 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
27227 void gc_heap::mark_absorb_new_alloc()
27229 fix_allocation_contexts (FALSE);
27231 gen0_bricks_cleared = FALSE;
27233 clear_gen0_bricks();
27236 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
27238 BOOL success = FALSE;
27239 BOOL thread_created = FALSE;
27240 dprintf (2, ("Preparing gc thread"));
27241 gh->bgc_threads_timeout_cs.Enter();
27242 if (!(gh->bgc_thread_running))
27244 dprintf (2, ("GC thread not runnning"));
27245 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
27248 thread_created = TRUE;
27253 dprintf (3, ("GC thread already running"));
27256 gh->bgc_threads_timeout_cs.Leave();
27259 FIRE_EVENT(GCCreateConcurrentThread_V1);
27264 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
27266 assert (background_gc_done_event.IsValid());
27268 //dprintf (2, ("Creating BGC thread"));
27270 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
27271 return gh->bgc_thread_running;
27274 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
27277 dprintf (3, ("Creating concurrent GC thread for the first time"));
27278 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
27282 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
27286 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
27290 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
27295 #ifdef MULTIPLE_HEAPS
27296 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
27298 UNREFERENCED_PARAMETER(number_of_heaps);
27299 #endif //MULTIPLE_HEAPS
27307 if (background_gc_done_event.IsValid())
27309 background_gc_done_event.CloseEvent();
27311 if (bgc_threads_sync_event.IsValid())
27313 bgc_threads_sync_event.CloseEvent();
27315 if (ee_proceed_event.IsValid())
27317 ee_proceed_event.CloseEvent();
27319 if (bgc_start_event.IsValid())
27321 bgc_start_event.CloseEvent();
27328 BOOL gc_heap::create_bgc_thread_support()
27333 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
27338 //needs to have room for enough smallest objects fitting on a page
27339 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
27345 make_c_mark_list (parr);
27353 if (gc_lh_block_event.IsValid())
27355 gc_lh_block_event.CloseEvent();
27362 int gc_heap::check_for_ephemeral_alloc()
27364 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
27368 #ifdef MULTIPLE_HEAPS
27369 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
27370 #endif //MULTIPLE_HEAPS
27372 for (int i = 0; i <= (max_generation - 1); i++)
27374 #ifdef MULTIPLE_HEAPS
27375 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
27377 if (get_new_allocation (i) <= 0)
27378 #endif //MULTIPLE_HEAPS
27380 gen = max (gen, i);
27391 // Wait for gc to finish sequential part
27392 void gc_heap::wait_to_proceed()
27394 assert (background_gc_done_event.IsValid());
27395 assert (bgc_start_event.IsValid());
27397 user_thread_wait(&ee_proceed_event, FALSE);
27400 // Start a new concurrent gc
27401 void gc_heap::start_c_gc()
27403 assert (background_gc_done_event.IsValid());
27404 assert (bgc_start_event.IsValid());
27406 //Need to make sure that the gc thread is in the right place.
27407 background_gc_done_event.Wait(INFINITE, FALSE);
27408 background_gc_done_event.Reset();
27409 bgc_start_event.Set();
27412 void gc_heap::do_background_gc()
27414 dprintf (2, ("starting a BGC"));
27415 #ifdef MULTIPLE_HEAPS
27416 for (int i = 0; i < n_heaps; i++)
27418 g_heaps[i]->init_background_gc();
27421 init_background_gc();
27422 #endif //MULTIPLE_HEAPS
27423 //start the background gc
27426 //wait until we get restarted by the BGC.
27430 void gc_heap::kill_gc_thread()
27432 //assert (settings.concurrent == FALSE);
27434 // We are doing a two-stage shutdown now.
27435 // In the first stage, we do minimum work, and call ExitProcess at the end.
27436 // In the secodn stage, we have the Loader lock and only one thread is
27437 // alive. Hence we do not need to kill gc thread.
27438 background_gc_done_event.CloseEvent();
27439 gc_lh_block_event.CloseEvent();
27440 bgc_start_event.CloseEvent();
27441 bgc_threads_timeout_cs.Destroy();
27443 recursive_gc_sync::shutdown();
27446 void gc_heap::bgc_thread_function()
27448 assert (background_gc_done_event.IsValid());
27449 assert (bgc_start_event.IsValid());
27451 dprintf (3, ("gc_thread thread starting..."));
27453 BOOL do_exit = FALSE;
27455 bool cooperative_mode = true;
27456 bgc_thread_id.SetToCurrentThread();
27457 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
27460 // Wait for work to do...
27461 dprintf (3, ("bgc thread: waiting..."));
27463 cooperative_mode = enable_preemptive ();
27464 //current_thread->m_fPreemptiveGCDisabled = 0;
27466 uint32_t result = bgc_start_event.Wait(
27468 #ifdef MULTIPLE_HEAPS
27472 #endif //MULTIPLE_HEAPS
27474 #ifdef MULTIPLE_HEAPS
27478 #endif //MULTIPLE_HEAPS
27481 dprintf (2, ("gc thread: finished waiting"));
27483 // not calling disable_preemptive here 'cause we
27484 // can't wait for GC complete here - RestartEE will be called
27485 // when we've done the init work.
27487 if (result == WAIT_TIMEOUT)
27489 // Should join the bgc threads and terminate all of them
27491 dprintf (1, ("GC thread timeout"));
27492 bgc_threads_timeout_cs.Enter();
27493 if (!keep_bgc_threads_p)
27495 dprintf (2, ("GC thread exiting"));
27496 bgc_thread_running = FALSE;
27498 bgc_thread_id.Clear();
27501 bgc_threads_timeout_cs.Leave();
27506 dprintf (3, ("GC thread needed, not exiting"));
27510 // if we signal the thread with no concurrent work to do -> exit
27511 if (!settings.concurrent)
27513 dprintf (3, ("no concurrent GC needed, exiting"));
27519 recursive_gc_sync::begin_background();
27520 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
27521 generation_free_list_space (generation_of (max_generation)),
27522 generation_free_obj_space (generation_of (max_generation)),
27523 dd_fragmentation (dynamic_data_of (max_generation))));
27527 current_bgc_state = bgc_not_in_process;
27530 //trace_gc = FALSE;
27533 enable_preemptive ();
27534 #ifdef MULTIPLE_HEAPS
27535 bgc_t_join.join(this, gc_join_done);
27536 if (bgc_t_join.joined())
27537 #endif //MULTIPLE_HEAPS
27539 enter_spin_lock (&gc_lock);
27540 dprintf (SPINLOCK_LOG, ("bgc Egc"));
27542 bgc_start_event.Reset();
27544 #ifdef MULTIPLE_HEAPS
27545 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
27547 size_t desired_per_heap = 0;
27548 size_t total_desired = 0;
27551 for (int i = 0; i < n_heaps; i++)
27554 dd = hp->dynamic_data_of (gen);
27555 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
27556 if (temp_total_desired < total_desired)
27559 total_desired = (size_t)MAX_PTR;
27562 total_desired = temp_total_desired;
27565 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
27567 for (int i = 0; i < n_heaps; i++)
27569 hp = gc_heap::g_heaps[i];
27570 dd = hp->dynamic_data_of (gen);
27571 dd_desired_allocation (dd) = desired_per_heap;
27572 dd_gc_new_allocation (dd) = desired_per_heap;
27573 dd_new_allocation (dd) = desired_per_heap;
27576 #endif //MULTIPLE_HEAPS
27577 #ifdef MULTIPLE_HEAPS
27579 #endif //MULTIPLE_HEAPS
27581 c_write (settings.concurrent, FALSE);
27582 recursive_gc_sync::end_background();
27583 keep_bgc_threads_p = FALSE;
27584 background_gc_done_event.Set();
27586 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27587 leave_spin_lock (&gc_lock);
27588 #ifdef MULTIPLE_HEAPS
27589 dprintf(1, ("End of BGC - starting all BGC threads"));
27590 bgc_t_join.restart();
27591 #endif //MULTIPLE_HEAPS
27593 // We can't disable preempt here because there might've been a GC already
27594 // started and decided to do a BGC and waiting for a BGC thread to restart
27595 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27596 // to restart the VM so we deadlock.
27597 //gc_heap::disable_preemptive (true);
27600 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27602 dprintf (3, ("bgc_thread thread exiting"));
27606 #endif //BACKGROUND_GC
27608 //Clear the cards [start_card, end_card[
27609 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27611 if (start_card < end_card)
27613 size_t start_word = card_word (start_card);
27614 size_t end_word = card_word (end_card);
27615 if (start_word < end_word)
27617 // Figure out the bit positions of the cards within their words
27618 unsigned bits = card_bit (start_card);
27619 card_table [start_word] &= lowbits (~0, bits);
27620 for (size_t i = start_word+1; i < end_word; i++)
27621 card_table [i] = 0;
27622 bits = card_bit (end_card);
27623 // Don't write beyond end_card (and possibly uncommitted card table space).
27626 card_table [end_word] &= highbits (~0, bits);
27631 // If the start and end cards are in the same word, just clear the appropriate card
27632 // bits in that word.
27633 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27634 highbits (~0, card_bit (end_card)));
27636 #ifdef VERYSLOWDEBUG
27637 size_t card = start_card;
27638 while (card < end_card)
27640 assert (! (card_set_p (card)));
27643 #endif //VERYSLOWDEBUG
27644 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27645 start_card, (size_t)card_address (start_card),
27646 end_card, (size_t)card_address (end_card)));
27650 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27652 size_t start_card = card_of (align_on_card (start_address));
27653 size_t end_card = card_of (align_lower_card (end_address));
27654 clear_cards (start_card, end_card);
27657 // copy [srccard, ...[ to [dst_card, end_card[
27658 // This will set the same bit twice. Can be optimized.
27660 void gc_heap::copy_cards (size_t dst_card,
27665 // If the range is empty, this function is a no-op - with the subtlety that
27666 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27667 // outside the committed region. To avoid the access, leave early.
27668 if (!(dst_card < end_card))
27671 unsigned int srcbit = card_bit (src_card);
27672 unsigned int dstbit = card_bit (dst_card);
27673 size_t srcwrd = card_word (src_card);
27674 size_t dstwrd = card_word (dst_card);
27675 unsigned int srctmp = card_table[srcwrd];
27676 unsigned int dsttmp = card_table[dstwrd];
27678 for (size_t card = dst_card; card < end_card; card++)
27680 if (srctmp & (1 << srcbit))
27681 dsttmp |= 1 << dstbit;
27683 dsttmp &= ~(1 << dstbit);
27684 if (!(++srcbit % 32))
27686 srctmp = card_table[++srcwrd];
27692 if (srctmp & (1 << srcbit))
27693 dsttmp |= 1 << dstbit;
27696 if (!(++dstbit % 32))
27698 card_table[dstwrd] = dsttmp;
27700 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27703 card_bundle_set(cardw_card_bundle(dstwrd));
27708 dsttmp = card_table[dstwrd];
27713 card_table[dstwrd] = dsttmp;
27715 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27718 card_bundle_set(cardw_card_bundle(dstwrd));
27723 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27725 ptrdiff_t relocation_distance = src - dest;
27726 size_t start_dest_card = card_of (align_on_card (dest));
27727 size_t end_dest_card = card_of (dest + len - 1);
27728 size_t dest_card = start_dest_card;
27729 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27730 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27731 src_card, (size_t)src, dest_card, (size_t)dest));
27732 dprintf (3,(" %Ix->%Ix:%Ix[",
27733 (size_t)src+len, end_dest_card, (size_t)dest+len));
27735 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27736 dest, src, len, relocation_distance, (align_on_card (dest))));
27738 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27739 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27741 //First card has two boundaries
27742 if (start_dest_card != card_of (dest))
27744 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27745 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27747 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27748 (card_address (start_dest_card) + relocation_distance),
27749 card_of (card_address (start_dest_card) + relocation_distance),
27751 card_of (src + len - 1)));
27753 dprintf (3, ("setting card: %Ix", card_of (dest)));
27754 set_card (card_of (dest));
27758 if (card_set_p (card_of (src)))
27759 set_card (card_of (dest));
27762 copy_cards (dest_card, src_card, end_dest_card,
27763 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27765 //Last card has two boundaries.
27766 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27767 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27769 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27770 (card_address (end_dest_card) + relocation_distance),
27771 card_of (card_address (end_dest_card) + relocation_distance),
27775 dprintf (3, ("setting card: %Ix", end_dest_card));
27776 set_card (end_dest_card);
27779 if (card_set_p (card_of (src + len - 1)))
27780 set_card (end_dest_card);
27782 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27783 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27787 #ifdef BACKGROUND_GC
27788 // this does not need the Interlocked version of mark_array_set_marked.
27789 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27791 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27792 (size_t)src, (size_t)dest,
27793 (size_t)src+len, (size_t)dest+len));
27795 uint8_t* src_o = src;
27797 uint8_t* src_end = src + len;
27798 int align_const = get_alignment_constant (TRUE);
27799 ptrdiff_t reloc = dest - src;
27801 while (src_o < src_end)
27803 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27805 if (background_object_marked (src_o, TRUE))
27807 dest_o = src_o + reloc;
27809 //if (background_object_marked (dest_o, FALSE))
27811 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27812 // FATAL_GC_ERROR();
27815 background_mark (dest_o,
27816 background_saved_lowest_address,
27817 background_saved_highest_address);
27818 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27824 #endif //BACKGROUND_GC
27826 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27828 size_t new_current_brick = brick_of (o);
27829 set_brick (new_current_brick,
27830 (o - brick_address (new_current_brick)));
27831 size_t b = 1 + new_current_brick;
27832 size_t limit = brick_of (next_o);
27833 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27834 dprintf(3,("b:%Ix->%Ix-%Ix",
27835 new_current_brick, (size_t)o, (size_t)next_o));
27838 set_brick (b,(new_current_brick - b));
27843 // start can not be >= heap_segment_allocated for the segment.
27844 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27846 size_t brick = brick_of (start);
27848 //last_object == null -> no search shortcut needed
27849 if ((brick == brick_of (first_object) || (start <= first_object)))
27855 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27856 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27857 int brick_entry = 0;
27860 if (prev_brick < min_brick)
27864 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27868 assert (! ((brick_entry == 0)));
27869 prev_brick = (brick_entry + prev_brick);
27872 o = ((prev_brick < min_brick) ? first_object :
27873 brick_address (prev_brick) + brick_entry - 1);
27874 assert (o <= start);
27877 assert (Align (size (o)) >= Align (min_obj_size));
27878 uint8_t* next_o = o + Align (size (o));
27879 size_t curr_cl = (size_t)next_o / brick_size;
27880 size_t min_cl = (size_t)first_object / brick_size;
27882 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27884 unsigned int n_o = 1;
27887 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27889 while (next_o <= start)
27897 assert (Align (size (o)) >= Align (min_obj_size));
27898 next_o = o + Align (size (o));
27900 }while (next_o < next_b);
27902 if (((size_t)next_o / brick_size) != curr_cl)
27904 if (curr_cl >= min_cl)
27906 fix_brick_to_highest (o, next_o);
27908 curr_cl = (size_t) next_o / brick_size;
27910 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27913 size_t bo = brick_of (o);
27914 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27915 dprintf (3, ("%Id o, [%Ix-[%Ix",
27919 set_brick (bo, (o - brick_address(bo)));
27934 // Find the first non-zero card word between cardw and cardw_end.
27935 // The index of the word we find is returned in cardw.
27936 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27938 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27939 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27941 if (card_bundles_enabled())
27943 size_t cardb = cardw_card_bundle (cardw);
27944 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27947 // Find a non-zero bundle
27948 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27952 if (cardb == end_cardb)
27955 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27956 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27957 while ((card_word < card_word_end) && !(*card_word))
27962 if (card_word != card_word_end)
27964 cardw = (card_word - &card_table[0]);
27967 else if ((cardw <= card_bundle_cardw (cardb)) &&
27968 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27970 // a whole bundle was explored and is empty
27971 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27972 dd_collection_count (dynamic_data_of (0)),
27973 cardb, card_bundle_cardw (cardb),
27974 card_bundle_cardw (cardb+1)));
27975 card_bundle_clear (cardb);
27983 uint32_t* card_word = &card_table[cardw];
27984 uint32_t* card_word_end = &card_table [cardw_end];
27986 while (card_word < card_word_end)
27988 if ((*card_word) != 0)
27990 cardw = (card_word - &card_table [0]);
28002 #endif //CARD_BUNDLE
28004 // Find cards that are set between two points in a card table.
28006 // card_table : The card table.
28007 // card : [in/out] As input, the card to start searching from.
28008 // As output, the first card that's set.
28009 // card_word_end : The card word at which to stop looking.
28010 // end_card : [out] The last card which is set.
28011 BOOL gc_heap::find_card(uint32_t* card_table,
28013 size_t card_word_end,
28016 uint32_t* last_card_word;
28017 uint32_t card_word_value;
28018 uint32_t bit_position;
28020 // Find the first card which is set
28021 last_card_word = &card_table [card_word (card)];
28022 bit_position = card_bit (card);
28023 card_word_value = (*last_card_word) >> bit_position;
28024 if (!card_word_value)
28028 // Using the card bundle, go through the remaining card words between here and
28029 // card_word_end until we find one that is non-zero.
28030 size_t lcw = card_word(card) + 1;
28031 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
28037 last_card_word = &card_table [lcw];
28038 card_word_value = *last_card_word;
28041 #else //CARD_BUNDLE
28042 // Go through the remaining card words between here and card_word_end until we find
28043 // one that is non-zero.
28049 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
28050 if (last_card_word < &card_table [card_word_end])
28052 card_word_value = *last_card_word;
28056 // We failed to find any non-zero card words before we got to card_word_end
28059 #endif //CARD_BUNDLE
28063 // Look for the lowest bit set
28064 if (card_word_value)
28066 while (!(card_word_value & 1))
28069 card_word_value = card_word_value / 2;
28073 // card is the card word index * card size + the bit index within the card
28074 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
28078 // Keep going until we get to an un-set card.
28080 card_word_value = card_word_value / 2;
28082 // If we reach the end of the card word and haven't hit a 0 yet, start going
28083 // card word by card word until we get to one that's not fully set (0xFFFF...)
28084 // or we reach card_word_end.
28085 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
28089 card_word_value = *(++last_card_word);
28090 } while ((last_card_word < &card_table [card_word_end]) &&
28093 (card_word_value == (1 << card_word_width)-1)
28095 // if left shift count >= width of type,
28096 // gcc reports error.
28097 (card_word_value == ~0u)
28102 } while (card_word_value & 1);
28104 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
28106 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
28107 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
28112 //because of heap expansion, computing end is complicated.
28113 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
28115 if ((low >= heap_segment_mem (seg)) &&
28116 (low < heap_segment_allocated (seg)))
28119 return heap_segment_allocated (seg);
28123 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
28126 UNREFERENCED_PARAMETER(low);
28128 //when relocating, the fault line is the plan start of the younger
28129 //generation because the generation is promoted.
28130 if (relocating && (gen_number == (settings.condemned_generation + 1)))
28132 generation* gen = generation_of (gen_number - 1);
28133 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
28134 assert (gen_alloc);
28139 assert (gen_number > settings.condemned_generation);
28140 return generation_allocation_start (generation_of (gen_number - 1 ));
28146 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
28147 size_t& cg_pointers_found)
28150 if ((gc_low <= o) && (gc_high > o))
28154 #ifdef MULTIPLE_HEAPS
28157 gc_heap* hp = heap_of (o);
28160 if ((hp->gc_low <= o) &&
28167 #endif //MULTIPLE_HEAPS
28168 cg_pointers_found ++;
28169 dprintf (4, ("keep card live for %Ix", o));
28173 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
28174 size_t& cg_pointers_found,
28175 card_fn fn, uint8_t* nhigh,
28176 uint8_t* next_boundary)
28179 if ((gc_low <= *poo) && (gc_high > *poo))
28182 call_fn(fn) (poo THREAD_NUMBER_ARG);
28184 #ifdef MULTIPLE_HEAPS
28187 gc_heap* hp = heap_of_gc (*poo);
28190 if ((hp->gc_low <= *poo) &&
28191 (hp->gc_high > *poo))
28194 call_fn(fn) (poo THREAD_NUMBER_ARG);
28196 if ((fn == &gc_heap::relocate_address) ||
28197 ((hp->ephemeral_low <= *poo) &&
28198 (hp->ephemeral_high > *poo)))
28200 cg_pointers_found++;
28204 #endif //MULTIPLE_HEAPS
28205 if ((next_boundary <= *poo) && (nhigh > *poo))
28207 cg_pointers_found ++;
28208 dprintf (4, ("cg pointer %Ix found, %Id so far",
28209 (size_t)*poo, cg_pointers_found ));
28214 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
28215 size_t& cg_pointers_found,
28216 size_t& n_eph, size_t& n_card_set,
28217 size_t& card, size_t& end_card,
28218 BOOL& foundp, uint8_t*& start_address,
28219 uint8_t*& limit, size_t& n_cards_cleared)
28221 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
28222 dprintf (3, ("ct: %Id cg", cg_pointers_found));
28223 BOOL passed_end_card_p = FALSE;
28226 if (cg_pointers_found == 0)
28228 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
28229 dprintf(3,(" CC [%Ix, %Ix[ ",
28230 (size_t)card_address(card), (size_t)po));
28231 clear_cards (card, card_of(po));
28232 n_card_set -= (card_of (po) - card);
28233 n_cards_cleared += (card_of (po) - card);
28236 n_eph +=cg_pointers_found;
28237 cg_pointers_found = 0;
28238 card = card_of (po);
28239 if (card >= end_card)
28241 passed_end_card_p = TRUE;
28242 dprintf (3, ("card %Ix exceeding end_card %Ix",
28243 (size_t)card, (size_t)end_card));
28244 foundp = find_card (card_table, card, card_word_end, end_card);
28247 n_card_set+= end_card - card;
28248 start_address = card_address (card);
28249 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
28250 (size_t)card, (size_t)start_address,
28251 (size_t)card_address (end_card)));
28253 limit = min (end, card_address (end_card));
28255 assert (!((limit == card_address (end_card))&&
28256 card_set_p (end_card)));
28259 return passed_end_card_p;
28262 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
28264 #ifdef BACKGROUND_GC
28265 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
28266 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
28268 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
28269 PREFIX_ASSUME(soh_seg != NULL);
28273 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
28275 heap_segment_background_allocated (soh_seg),
28276 heap_segment_allocated (soh_seg)));
28278 soh_seg = heap_segment_next_rw (soh_seg);
28280 #endif //BACKGROUND_GC
28282 uint8_t* low = gc_low;
28283 uint8_t* high = gc_high;
28284 size_t end_card = 0;
28286 generation* oldest_gen = generation_of (max_generation);
28287 int curr_gen_number = max_generation;
28288 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
28289 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
28291 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
28292 PREFIX_ASSUME(seg != NULL);
28294 uint8_t* beg = generation_allocation_start (oldest_gen);
28295 uint8_t* end = compute_next_end (seg, low);
28296 uint8_t* last_object = beg;
28298 size_t cg_pointers_found = 0;
28300 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
28304 size_t n_card_set = 0;
28305 uint8_t* nhigh = (relocating ?
28306 heap_segment_plan_allocated (ephemeral_heap_segment) : high);
28308 BOOL foundp = FALSE;
28309 uint8_t* start_address = 0;
28310 uint8_t* limit = 0;
28311 size_t card = card_of (beg);
28312 #ifdef BACKGROUND_GC
28313 BOOL consider_bgc_mark_p = FALSE;
28314 BOOL check_current_sweep_p = FALSE;
28315 BOOL check_saved_sweep_p = FALSE;
28316 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28317 #endif //BACKGROUND_GC
28319 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
28320 size_t total_cards_cleared = 0;
28324 if (card_of(last_object) > card)
28326 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
28327 if (cg_pointers_found == 0)
28329 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
28330 clear_cards (card, card_of(last_object));
28331 n_card_set -= (card_of (last_object) - card);
28332 total_cards_cleared += (card_of (last_object) - card);
28335 n_eph += cg_pointers_found;
28336 cg_pointers_found = 0;
28337 card = card_of (last_object);
28340 if (card >= end_card)
28342 foundp = find_card (card_table, card, card_word_end, end_card);
28345 n_card_set += end_card - card;
28346 start_address = max (beg, card_address (card));
28348 limit = min (end, card_address (end_card));
28350 if (!foundp || (last_object >= end) || (card_address (card) >= end))
28352 if (foundp && (cg_pointers_found == 0))
28354 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
28356 clear_cards (card, card_of (end));
28357 n_card_set -= (card_of (end) - card);
28358 total_cards_cleared += (card_of (end) - card);
28360 n_eph += cg_pointers_found;
28361 cg_pointers_found = 0;
28362 if ((seg = heap_segment_next_in_range (seg)) != 0)
28364 #ifdef BACKGROUND_GC
28365 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28366 #endif //BACKGROUND_GC
28367 beg = heap_segment_mem (seg);
28368 end = compute_next_end (seg, low);
28369 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
28370 card = card_of (beg);
28381 assert (card_set_p (card));
28383 uint8_t* o = last_object;
28385 o = find_first_object (start_address, last_object);
28386 // Never visit an object twice.
28387 assert (o >= last_object);
28389 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
28390 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
28391 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
28395 assert (Align (size (o)) >= Align (min_obj_size));
28396 size_t s = size (o);
28398 uint8_t* next_o = o + Align (s);
28401 if ((o >= gen_boundary) &&
28402 (seg == ephemeral_heap_segment))
28404 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
28406 assert ((curr_gen_number > 0));
28407 gen_boundary = generation_allocation_start
28408 (generation_of (curr_gen_number - 1));
28409 next_boundary = (compute_next_boundary
28410 (low, curr_gen_number, relocating));
28413 dprintf (4, ("|%Ix|", (size_t)o));
28415 if (next_o < start_address)
28420 #ifdef BACKGROUND_GC
28421 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
28425 #endif //BACKGROUND_GC
28427 #ifdef COLLECTIBLE_CLASS
28428 if (is_collectible(o))
28430 BOOL passed_end_card_p = FALSE;
28432 if (card_of (o) > card)
28434 passed_end_card_p = card_transition (o, end, card_word_end,
28438 foundp, start_address,
28439 limit, total_cards_cleared);
28442 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
28444 // card is valid and it covers the head of the object
28445 if (fn == &gc_heap::relocate_address)
28447 keep_card_live (o, n_gen, cg_pointers_found);
28451 uint8_t* class_obj = get_class_object (o);
28452 mark_through_cards_helper (&class_obj, n_gen,
28453 cg_pointers_found, fn,
28454 nhigh, next_boundary);
28458 if (passed_end_card_p)
28460 if (foundp && (card_address (card) < next_o))
28462 goto go_through_refs;
28464 else if (foundp && (start_address < limit))
28466 next_o = find_first_object (start_address, o);
28475 #endif //COLLECTIBLE_CLASS
28477 if (contain_pointers (o))
28479 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
28482 dprintf (4, ("normal object path"));
28484 (method_table(o), o, s, poo,
28485 start_address, use_start, (o + s),
28487 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
28488 if (card_of ((uint8_t*)poo) > card)
28490 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
28495 foundp, start_address,
28496 limit, total_cards_cleared);
28498 if (passed_end_card_p)
28500 if (foundp && (card_address (card) < next_o))
28504 if (ppstop <= (uint8_t**)start_address)
28506 else if (poo < (uint8_t**)start_address)
28507 {poo = (uint8_t**)start_address;}
28510 else if (foundp && (start_address < limit))
28512 next_o = find_first_object (start_address, o);
28520 mark_through_cards_helper (poo, n_gen,
28521 cg_pointers_found, fn,
28522 nhigh, next_boundary);
28529 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
28531 if (brick_table [brick_of (o)] <0)
28532 fix_brick_to_highest (o, next_o);
28540 // compute the efficiency ratio of the card table
28543 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
28544 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28545 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
28549 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28550 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
28554 #ifdef SEG_REUSE_STATS
28555 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
28557 size_t total_items = 0;
28559 for (int i = 0; i < count; i++)
28561 total_items += ordered_indices[i];
28562 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
28563 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
28565 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
28566 return total_items;
28568 #endif // SEG_REUSE_STATS
28570 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
28572 // detect pinned plugs
28573 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28575 deque_pinned_plug();
28576 update_oldest_pinned_plug();
28577 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28581 size_t plug_size = last_plug_size + Align(min_obj_size);
28582 BOOL is_padded = FALSE;
28585 plug_size += Align (min_obj_size);
28587 #endif //SHORT_PLUGS
28589 #ifdef RESPECT_LARGE_ALIGNMENT
28590 plug_size += switch_alignment_size (is_padded);
28591 #endif //RESPECT_LARGE_ALIGNMENT
28593 total_ephemeral_plugs += plug_size;
28594 size_t plug_size_power2 = round_up_power2 (plug_size);
28595 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28596 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28600 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28604 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28606 assert ((tree != NULL));
28607 if (node_left_child (tree))
28609 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28612 if (last_plug != 0)
28614 uint8_t* plug = tree;
28615 size_t gap_size = node_gap_size (plug);
28616 uint8_t* gap = (plug - gap_size);
28617 uint8_t* last_plug_end = gap;
28618 size_t last_plug_size = (last_plug_end - last_plug);
28619 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28620 tree, last_plug, gap_size, gap, last_plug_size));
28622 if (tree == oldest_pinned_plug)
28624 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28625 tree, last_plug, last_plug_size));
28626 mark* m = oldest_pin();
28627 if (m->has_pre_plug_info())
28629 last_plug_size += sizeof (gap_reloc_pair);
28630 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28633 // Can't assert here - if it's a pinned plug it can be less.
28634 //assert (last_plug_size >= Align (min_obj_size));
28636 count_plug (last_plug_size, last_plug);
28641 if (node_right_child (tree))
28643 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28647 void gc_heap::build_ordered_plug_indices ()
28649 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28650 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28652 uint8_t* start_address = generation_limit (max_generation);
28653 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28654 size_t current_brick = brick_of (start_address);
28655 size_t end_brick = brick_of (end_address - 1);
28656 uint8_t* last_plug = 0;
28658 //Look for the right pinned plug to start from.
28659 reset_pinned_queue_bos();
28660 while (!pinned_plug_que_empty_p())
28662 mark* m = oldest_pin();
28663 if ((m->first >= start_address) && (m->first < end_address))
28665 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28670 deque_pinned_plug();
28673 update_oldest_pinned_plug();
28675 while (current_brick <= end_brick)
28677 int brick_entry = brick_table [ current_brick ];
28678 if (brick_entry >= 0)
28680 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28688 count_plug (end_address - last_plug, last_plug);
28691 // we need to make sure that after fitting all the existing plugs, we
28692 // have big enough free space left to guarantee that the next allocation
28694 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28695 total_ephemeral_plugs += extra_size;
28696 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28697 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28699 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28701 #ifdef SEG_REUSE_STATS
28702 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28703 size_t total_plug_power2 = 0;
28704 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28705 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28706 total_ephemeral_plugs,
28708 (total_ephemeral_plugs ?
28709 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28711 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28712 #endif // SEG_REUSE_STATS
28715 void gc_heap::init_ordered_free_space_indices ()
28717 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28718 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28721 void gc_heap::trim_free_spaces_indices ()
28723 trimmed_free_space_index = -1;
28724 size_t max_count = max_free_space_items - 1;
28727 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28729 count += ordered_free_space_indices[i];
28731 if (count >= max_count)
28737 ptrdiff_t extra_free_space_items = count - max_count;
28739 if (extra_free_space_items > 0)
28741 ordered_free_space_indices[i] -= extra_free_space_items;
28742 free_space_items = max_count;
28743 trimmed_free_space_index = i;
28747 free_space_items = count;
28755 free_space_buckets = MAX_NUM_BUCKETS - i;
28757 for (--i; i >= 0; i--)
28759 ordered_free_space_indices[i] = 0;
28762 memcpy (saved_ordered_free_space_indices,
28763 ordered_free_space_indices,
28764 sizeof(ordered_free_space_indices));
28767 // We fit as many plugs as we can and update the number of plugs left and the number
28768 // of free spaces left.
28769 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28771 assert (small_index <= big_index);
28772 assert (big_index < MAX_NUM_BUCKETS);
28774 size_t small_blocks = ordered_blocks[small_index];
28776 if (small_blocks == 0)
28781 size_t big_spaces = ordered_spaces[big_index];
28783 if (big_spaces == 0)
28788 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28790 small_blocks, (small_index + MIN_INDEX_POWER2),
28791 big_spaces, (big_index + MIN_INDEX_POWER2)));
28793 size_t big_to_small = big_spaces << (big_index - small_index);
28795 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28796 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28798 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28799 BOOL can_fit = (extra_small_spaces >= 0);
28803 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28805 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28810 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28811 ordered_spaces[big_index] = 0;
28812 if (extra_small_spaces > 0)
28814 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28815 ordered_blocks[small_index] = 0;
28816 for (i = small_index; i < big_index; i++)
28818 if (extra_small_spaces & 1)
28820 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28822 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28823 ordered_spaces[i] += 1;
28825 extra_small_spaces >>= 1;
28828 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28830 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28831 ordered_spaces[i] += extra_small_spaces;
28835 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28837 (small_index + MIN_INDEX_POWER2),
28838 ordered_blocks[small_index],
28839 (ordered_blocks[small_index] - big_to_small)));
28840 ordered_blocks[small_index] -= big_to_small;
28843 #ifdef SEG_REUSE_STATS
28845 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28846 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28848 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28849 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28850 #endif //SEG_REUSE_STATS
28855 // space_index gets updated to the biggest available space index.
28856 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28858 assert (*space_index >= block_index);
28860 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28863 if (*space_index < block_index)
28872 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28874 #ifdef FEATURE_STRUCTALIGN
28875 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28877 #endif // FEATURE_STRUCTALIGN
28878 int space_index = count - 1;
28879 for (int block_index = (count - 1); block_index >= 0; block_index--)
28881 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28890 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28892 assert (bestfit_seg);
28894 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28895 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28896 // free_space_buckets,
28897 // free_space_items);
28899 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28900 ordered_free_space_indices,
28904 assert (settings.condemned_generation == max_generation);
28906 uint8_t* first_address = heap_segment_mem (seg);
28907 uint8_t* end_address = heap_segment_reserved (seg);
28908 //look through the pinned plugs for relevant ones.
28909 //Look for the right pinned plug to start from.
28910 reset_pinned_queue_bos();
28912 // See comment in can_expand_into_p why we need (max_generation + 1).
28913 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28914 BOOL has_fit_gen_starts = FALSE;
28916 while (!pinned_plug_que_empty_p())
28919 if ((pinned_plug (m) >= first_address) &&
28920 (pinned_plug (m) < end_address) &&
28921 (pinned_len (m) >= eph_gen_starts))
28924 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28929 deque_pinned_plug();
28933 if (!pinned_plug_que_empty_p())
28935 bestfit_seg->add ((void*)m, TRUE, TRUE);
28936 deque_pinned_plug();
28938 has_fit_gen_starts = TRUE;
28941 while (!pinned_plug_que_empty_p() &&
28942 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28944 bestfit_seg->add ((void*)m, TRUE, FALSE);
28945 deque_pinned_plug();
28949 if (commit_end_of_seg)
28951 if (!has_fit_gen_starts)
28953 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28955 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28959 bestfit_seg->check();
28963 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28965 if (!end_of_segment_p)
28967 trim_free_spaces_indices ();
28970 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28971 ordered_free_space_indices,
28974 return can_bestfit;
28977 BOOL gc_heap::best_fit (size_t free_space,
28978 size_t largest_free_space,
28979 size_t additional_space,
28980 BOOL* use_additional_space)
28982 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28984 assert (!additional_space || (additional_space && use_additional_space));
28985 if (use_additional_space)
28987 *use_additional_space = FALSE;
28990 if (ordered_plug_indices_init == FALSE)
28992 total_ephemeral_plugs = 0;
28993 build_ordered_plug_indices();
28994 ordered_plug_indices_init = TRUE;
28998 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
29001 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
29003 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
29004 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
29005 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
29006 if (!can_fit_empty_eph)
29008 can_fit_empty_eph = (additional_space >= empty_eph);
29010 if (can_fit_empty_eph)
29012 *use_additional_space = TRUE;
29016 return can_fit_empty_eph;
29019 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
29021 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
29025 if ((free_space + additional_space) == 0)
29027 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
29031 #ifdef SEG_REUSE_STATS
29032 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
29033 size_t total_free_space_power2 = 0;
29034 size_t total_free_space_items =
29035 dump_buckets (ordered_free_space_indices,
29037 &total_free_space_power2);
29038 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
29040 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
29041 total_ephemeral_plugs,
29043 total_free_space_power2,
29044 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
29045 additional_space));
29047 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
29048 memcpy (saved_all_free_space_indices,
29049 ordered_free_space_indices,
29050 sizeof(saved_all_free_space_indices));
29052 #endif // SEG_REUSE_STATS
29054 if (total_ephemeral_plugs > (free_space + additional_space))
29059 use_bestfit = try_best_fit(FALSE);
29061 if (!use_bestfit && additional_space)
29063 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
29065 if (relative_free_space_index != -1)
29067 int relative_plug_index = 0;
29068 size_t plugs_to_fit = 0;
29070 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
29072 plugs_to_fit = ordered_plug_indices[relative_plug_index];
29073 if (plugs_to_fit != 0)
29079 if ((relative_plug_index > relative_free_space_index) ||
29080 ((relative_plug_index == relative_free_space_index) &&
29081 (plugs_to_fit > 1)))
29083 #ifdef SEG_REUSE_STATS
29084 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
29085 (relative_free_space_index + MIN_INDEX_POWER2),
29087 (relative_plug_index + MIN_INDEX_POWER2)));
29088 #endif // SEG_REUSE_STATS
29092 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
29093 ordered_free_space_indices[relative_free_space_index]++;
29094 use_bestfit = try_best_fit(TRUE);
29097 free_space_items++;
29098 // Since we might've trimmed away some of the free spaces we had, we should see
29099 // if we really need to use end of seg space - if it's the same or smaller than
29100 // the largest space we trimmed we can just add that one back instead of
29101 // using end of seg.
29102 if (relative_free_space_index > trimmed_free_space_index)
29104 *use_additional_space = TRUE;
29108 // If the addition space is <= than the last trimmed space, we
29109 // should just use that last trimmed space instead.
29110 saved_ordered_free_space_indices[trimmed_free_space_index]++;
29120 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
29122 #ifdef SEG_REUSE_STATS
29123 size_t saved_max = max_free_space_items;
29124 BOOL temp_bestfit = FALSE;
29126 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
29127 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
29129 // TODO: need to take the end of segment into consideration.
29130 while (max_free_space_items <= total_free_space_items)
29132 max_free_space_items += max_free_space_items / 2;
29133 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
29134 memcpy (ordered_free_space_indices,
29135 saved_all_free_space_indices,
29136 sizeof(ordered_free_space_indices));
29137 if (try_best_fit(FALSE))
29139 temp_bestfit = TRUE;
29146 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
29150 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
29153 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
29154 max_free_space_items = saved_max;
29155 #endif // SEG_REUSE_STATS
29156 if (free_space_items)
29158 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
29159 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
29163 max_free_space_items = MAX_NUM_FREE_SPACES;
29167 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
29168 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
29170 return use_bestfit;
29173 BOOL gc_heap::process_free_space (heap_segment* seg,
29175 size_t min_free_size,
29176 size_t min_cont_size,
29177 size_t* total_free_space,
29178 size_t* largest_free_space)
29180 *total_free_space += free_space;
29181 *largest_free_space = max (*largest_free_space, free_space);
29183 #ifdef SIMPLE_DPRINTF
29184 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
29185 free_space, *total_free_space, *largest_free_space));
29186 #endif //SIMPLE_DPRINTF
29188 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
29190 #ifdef SIMPLE_DPRINTF
29191 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
29192 settings.condemned_generation,
29193 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
29196 UNREFERENCED_PARAMETER(seg);
29197 #endif //SIMPLE_DPRINTF
29201 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
29202 if (free_space_index != -1)
29204 ordered_free_space_indices[free_space_index]++;
29209 BOOL gc_heap::expand_reused_seg_p()
29211 BOOL reused_seg = FALSE;
29212 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
29213 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
29214 (heap_expand_mechanism == expand_reuse_normal))
29222 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
29223 allocator* gen_allocator)
29225 min_cont_size += END_SPACE_AFTER_GC;
29226 use_bestfit = FALSE;
29227 commit_end_of_seg = FALSE;
29228 bestfit_first_pin = 0;
29229 uint8_t* first_address = heap_segment_mem (seg);
29230 uint8_t* end_address = heap_segment_reserved (seg);
29231 size_t end_extra_space = end_space_after_gc();
29233 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
29235 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
29236 first_address, end_address, end_extra_space));
29240 end_address -= end_extra_space;
29242 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
29243 settings.condemned_generation, min_free_size, min_cont_size));
29244 size_t eph_gen_starts = eph_gen_starts_size;
29246 if (settings.condemned_generation == max_generation)
29248 size_t free_space = 0;
29249 size_t largest_free_space = free_space;
29250 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
29251 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
29252 //We are going to allocate the generation starts in the 1st free space,
29253 //so start from the first free space that's big enough for gen starts and a min object size.
29254 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
29255 // we could use it by allocating the last generation start a bit bigger but
29256 // the complexity isn't worth the effort (those plugs are from gen2
29257 // already anyway).
29258 reset_pinned_queue_bos();
29260 BOOL has_fit_gen_starts = FALSE;
29262 init_ordered_free_space_indices ();
29263 while (!pinned_plug_que_empty_p())
29266 if ((pinned_plug (m) >= first_address) &&
29267 (pinned_plug (m) < end_address) &&
29268 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
29274 deque_pinned_plug();
29278 if (!pinned_plug_que_empty_p())
29280 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
29282 if (process_free_space (seg,
29283 pinned_len (m) - eph_gen_starts,
29284 min_free_size, min_cont_size,
29285 &free_space, &largest_free_space))
29290 deque_pinned_plug();
29292 has_fit_gen_starts = TRUE;
29295 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
29297 //tally up free space
29298 while (!pinned_plug_que_empty_p() &&
29299 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
29301 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
29302 if (process_free_space (seg,
29304 min_free_size, min_cont_size,
29305 &free_space, &largest_free_space))
29310 deque_pinned_plug();
29314 //try to find space at the end of the segment.
29315 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
29316 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
29317 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
29318 if (end_space >= additional_space)
29320 BOOL can_fit = TRUE;
29321 commit_end_of_seg = TRUE;
29323 if (largest_free_space < min_cont_size)
29325 if (end_space >= min_cont_size)
29327 additional_space = max (min_cont_size, additional_space);
29328 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
29333 if (settings.concurrent)
29336 commit_end_of_seg = FALSE;
29340 size_t additional_space_bestfit = additional_space;
29341 if (!has_fit_gen_starts)
29343 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
29345 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
29346 additional_space_bestfit));
29350 bestfit_first_pin = heap_segment_plan_allocated (seg);
29351 additional_space_bestfit -= eph_gen_starts;
29354 can_fit = best_fit (free_space,
29355 largest_free_space,
29356 additional_space_bestfit,
29357 &commit_end_of_seg);
29361 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
29362 seg, (commit_end_of_seg ? "with" : "without")));
29366 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29373 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
29376 assert (additional_space <= end_space);
29377 if (commit_end_of_seg)
29379 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
29381 dprintf (2, ("Couldn't commit end of segment?!"));
29382 use_bestfit = FALSE;
29389 // We increase the index here because growing heap segment could create a discrepency with
29390 // the additional space we used (could be bigger).
29391 size_t free_space_end_of_seg =
29392 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
29393 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
29394 saved_ordered_free_space_indices[relative_free_space_index]++;
29400 memcpy (ordered_free_space_indices,
29401 saved_ordered_free_space_indices,
29402 sizeof(ordered_free_space_indices));
29403 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
29404 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
29405 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
29411 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29416 assert (settings.condemned_generation == (max_generation-1));
29417 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
29418 size_t largest_free_space = free_space;
29419 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
29420 //find the first free list in range of the current segment
29421 size_t sz_list = gen_allocator->first_bucket_size();
29422 unsigned int a_l_idx = 0;
29423 uint8_t* free_list = 0;
29424 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
29426 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
29428 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29431 if ((free_list >= first_address) &&
29432 (free_list < end_address) &&
29433 (unused_array_size (free_list) >= eph_gen_starts))
29439 free_list = free_list_slot (free_list);
29447 init_ordered_free_space_indices ();
29448 if (process_free_space (seg,
29449 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
29450 min_free_size, min_cont_size,
29451 &free_space, &largest_free_space))
29456 free_list = free_list_slot (free_list);
29460 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
29464 //tally up free space
29470 if ((free_list >= first_address) && (free_list < end_address) &&
29471 process_free_space (seg,
29472 unused_array_size (free_list),
29473 min_free_size, min_cont_size,
29474 &free_space, &largest_free_space))
29479 free_list = free_list_slot (free_list);
29482 if (a_l_idx < gen_allocator->number_of_buckets())
29484 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29490 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29494 BOOL can_fit = best_fit (free_space, 0, NULL);
29497 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
29501 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29509 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
29510 generation* gen, uint8_t* start_address,
29511 unsigned int& active_new_gen_number,
29512 uint8_t*& last_pinned_gap, BOOL& leftp,
29515 , mark* pinned_plug_entry
29516 #endif //SHORT_PLUGS
29519 // detect generation boundaries
29520 // make sure that active_new_gen_number is not the youngest generation.
29521 // because the generation_limit wouldn't return the right thing in this case.
29524 if ((active_new_gen_number > 1) &&
29525 (last_plug >= generation_limit (active_new_gen_number)))
29527 assert (last_plug >= start_address);
29528 active_new_gen_number--;
29529 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
29530 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
29535 // detect pinned plugs
29536 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
29538 size_t entry = deque_pinned_plug();
29539 mark* m = pinned_plug_of (entry);
29541 size_t saved_pinned_len = pinned_len(m);
29542 pinned_len(m) = last_plug - last_pinned_gap;
29543 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
29545 if (m->has_post_plug_info())
29547 last_plug_size += sizeof (gap_reloc_pair);
29548 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29551 last_pinned_gap = last_plug + last_plug_size;
29552 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
29553 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
29556 //we are creating a generation fault. set the cards.
29558 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
29559 size_t card = card_of (last_plug);
29560 while (card != end_card)
29567 else if (last_plug >= start_address)
29569 #ifdef FEATURE_STRUCTALIGN
29570 int requiredAlignment;
29572 node_aligninfo (last_plug, requiredAlignment, pad);
29574 // from how we previously aligned the plug's destination address,
29575 // compute the actual alignment offset.
29576 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
29577 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
29578 if (!alignmentOffset)
29580 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29581 alignmentOffset = requiredAlignment;
29584 //clear the alignment info because we are reallocating
29585 clear_node_aligninfo (last_plug);
29586 #else // FEATURE_STRUCTALIGN
29587 //clear the realignment flag because we are reallocating
29588 clear_node_realigned (last_plug);
29589 #endif // FEATURE_STRUCTALIGN
29590 BOOL adjacentp = FALSE;
29591 BOOL set_padding_on_saved_p = FALSE;
29595 last_plug_size += sizeof (gap_reloc_pair);
29598 assert (pinned_plug_entry != NULL);
29599 if (last_plug_size <= sizeof (plug_and_gap))
29601 set_padding_on_saved_p = TRUE;
29603 #endif //SHORT_PLUGS
29605 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29609 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29610 #endif //SHORT_PLUGS
29612 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29614 set_padding_on_saved_p,
29616 #endif //SHORT_PLUGS
29617 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29619 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29620 assert (new_address);
29621 set_node_relocation_distance (last_plug, new_address - last_plug);
29622 #ifdef FEATURE_STRUCTALIGN
29623 if (leftp && node_alignpad (last_plug) == 0)
29624 #else // FEATURE_STRUCTALIGN
29625 if (leftp && !node_realigned (last_plug))
29626 #endif // FEATURE_STRUCTALIGN
29628 // TODO - temporarily disable L optimization because of a bug in it.
29629 //set_node_left (last_plug);
29631 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29636 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29637 uint8_t* start_address,
29639 unsigned int& active_new_gen_number,
29640 uint8_t*& last_pinned_gap, BOOL& leftp)
29642 assert (tree != NULL);
29643 int left_node = node_left_child (tree);
29644 int right_node = node_right_child (tree);
29646 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29647 tree, last_pinned_gap, last_plug, left_node, right_node));
29651 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29652 realloc_in_brick ((tree + left_node), last_plug, start_address,
29653 gen, active_new_gen_number, last_pinned_gap,
29657 if (last_plug != 0)
29659 uint8_t* plug = tree;
29661 BOOL has_pre_plug_info_p = FALSE;
29662 BOOL has_post_plug_info_p = FALSE;
29663 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29664 &has_pre_plug_info_p,
29665 &has_post_plug_info_p,
29668 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29669 // The pinned plugs are handled in realloc_plug.
29670 size_t gap_size = node_gap_size (plug);
29671 uint8_t* gap = (plug - gap_size);
29672 uint8_t* last_plug_end = gap;
29673 size_t last_plug_size = (last_plug_end - last_plug);
29674 // Cannot assert this - a plug could be less than that due to the shortened ones.
29675 //assert (last_plug_size >= Align (min_obj_size));
29676 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29677 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29678 realloc_plug (last_plug_size, last_plug, gen, start_address,
29679 active_new_gen_number, last_pinned_gap,
29680 leftp, has_pre_plug_info_p
29682 , pinned_plug_entry
29683 #endif //SHORT_PLUGS
29691 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29692 realloc_in_brick ((tree + right_node), last_plug, start_address,
29693 gen, active_new_gen_number, last_pinned_gap,
29699 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29700 uint8_t* start_address, uint8_t* end_address,
29701 unsigned active_new_gen_number)
29703 dprintf (3, ("--- Reallocing ---"));
29707 //make sure that every generation has a planned allocation start
29708 int gen_number = max_generation - 1;
29709 while (gen_number >= 0)
29711 generation* gen = generation_of (gen_number);
29712 if (0 == generation_plan_allocation_start (gen))
29714 generation_plan_allocation_start (gen) =
29715 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29716 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29717 assert (generation_plan_allocation_start (gen));
29723 uint8_t* first_address = start_address;
29724 //Look for the right pinned plug to start from.
29725 reset_pinned_queue_bos();
29726 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29727 while (!pinned_plug_que_empty_p())
29729 mark* m = oldest_pin();
29730 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29732 if (pinned_plug (m) < first_address)
29734 first_address = pinned_plug (m);
29739 deque_pinned_plug();
29742 size_t current_brick = brick_of (first_address);
29743 size_t end_brick = brick_of (end_address-1);
29744 uint8_t* last_plug = 0;
29746 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29747 BOOL leftp = FALSE;
29749 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29750 start_address, first_address, pinned_plug (oldest_pin())));
29752 while (current_brick <= end_brick)
29754 int brick_entry = brick_table [ current_brick ];
29755 if (brick_entry >= 0)
29757 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29758 last_plug, start_address, consing_gen,
29759 active_new_gen_number, last_pinned_gap,
29765 if (last_plug != 0)
29767 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29769 active_new_gen_number, last_pinned_gap,
29773 #endif //SHORT_PLUGS
29777 //Fix the old segment allocated size
29778 assert (last_pinned_gap >= heap_segment_mem (seg));
29779 assert (last_pinned_gap <= heap_segment_committed (seg));
29780 heap_segment_plan_allocated (seg) = last_pinned_gap;
29783 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29786 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29788 BOOL contains_pinned_plugs = FALSE;
29791 while (mi != mark_stack_tos)
29793 m = pinned_plug_of (mi);
29794 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29796 contains_pinned_plugs = TRUE;
29803 if (contains_pinned_plugs)
29808 #endif //VERIFY_HEAP
29811 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29813 if (!should_expand_in_full_gc)
29815 if ((condemned_gen_number != max_generation) &&
29816 (settings.pause_mode != pause_low_latency) &&
29817 (settings.pause_mode != pause_sustained_low_latency))
29819 should_expand_in_full_gc = TRUE;
29824 void gc_heap::save_ephemeral_generation_starts()
29826 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29828 saved_ephemeral_plan_start[ephemeral_generation] =
29829 generation_plan_allocation_start (generation_of (ephemeral_generation));
29830 saved_ephemeral_plan_start_size[ephemeral_generation] =
29831 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29835 generation* gc_heap::expand_heap (int condemned_generation,
29836 generation* consing_gen,
29837 heap_segment* new_heap_segment)
29839 UNREFERENCED_PARAMETER(condemned_generation);
29840 assert (condemned_generation >= (max_generation -1));
29841 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29842 uint8_t* start_address = generation_limit (max_generation);
29843 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29844 BOOL should_promote_ephemeral = FALSE;
29845 ptrdiff_t eph_size = total_ephemeral_size;
29846 #ifdef BACKGROUND_GC
29847 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29848 #endif //BACKGROUND_GC
29849 settings.heap_expansion = TRUE;
29851 #ifdef BACKGROUND_GC
29852 if (cm_in_progress)
29854 if (!expanded_in_fgc)
29856 expanded_in_fgc = TRUE;
29859 #endif //BACKGROUND_GC
29861 //reset the elevation state for next time.
29862 dprintf (2, ("Elevation: elevation = el_none"));
29863 if (settings.should_lock_elevation && !expand_reused_seg_p())
29864 settings.should_lock_elevation = FALSE;
29866 heap_segment* new_seg = new_heap_segment;
29869 return consing_gen;
29871 //copy the card and brick tables
29872 if (g_gc_card_table!= card_table)
29873 copy_brick_card_table();
29875 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29876 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29878 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29879 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29880 heap_segment_mem (ephemeral_heap_segment));
29881 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29882 heap_segment_committed (ephemeral_heap_segment));
29884 assert (generation_plan_allocation_start (youngest_generation));
29885 assert (generation_plan_allocation_start (youngest_generation) <
29886 heap_segment_plan_allocated (ephemeral_heap_segment));
29888 if (settings.pause_mode == pause_no_gc)
29890 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29891 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29892 should_promote_ephemeral = TRUE;
29898 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29902 if (should_promote_ephemeral)
29904 ephemeral_promotion = TRUE;
29905 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29906 dprintf (2, ("promoting ephemeral"));
29907 save_ephemeral_generation_starts();
29911 // commit the new ephemeral segment all at once if it is a new one.
29912 if ((eph_size > 0) && new_segment_p)
29914 #ifdef FEATURE_STRUCTALIGN
29915 // The destination may require a larger alignment padding than the source.
29916 // Assume the worst possible alignment padding.
29917 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29918 #endif // FEATURE_STRUCTALIGN
29919 #ifdef RESPECT_LARGE_ALIGNMENT
29920 //Since the generation start can be larger than min_obj_size
29921 //The alignment could be switched.
29922 eph_size += switch_alignment_size(FALSE);
29923 #endif //RESPECT_LARGE_ALIGNMENT
29924 //Since the generation start can be larger than min_obj_size
29925 //Compare the alignment of the first object in gen1
29926 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29928 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29929 return consing_gen;
29931 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29934 //Fix the end of the old ephemeral heap segment
29935 heap_segment_plan_allocated (ephemeral_heap_segment) =
29936 generation_plan_allocation_start (generation_of (max_generation-1));
29938 dprintf (3, ("Old ephemeral allocated set to %Ix",
29939 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29944 // TODO - Is this really necessary? We should think about it.
29945 //initialize the first brick
29946 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29947 set_brick (first_brick,
29948 heap_segment_mem (new_seg) - brick_address (first_brick));
29951 //From this point on, we cannot run out of memory
29953 //reset the allocation of the consing generation back to the end of the
29954 //old ephemeral segment
29955 generation_allocation_limit (consing_gen) =
29956 heap_segment_plan_allocated (ephemeral_heap_segment);
29957 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29958 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29960 //clear the generation gap for all of the ephemeral generations
29962 int generation_num = max_generation-1;
29963 while (generation_num >= 0)
29965 generation* gen = generation_of (generation_num);
29966 generation_plan_allocation_start (gen) = 0;
29971 heap_segment* old_seg = ephemeral_heap_segment;
29972 ephemeral_heap_segment = new_seg;
29974 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29975 //because the relocation and compact phases shouldn't see it
29977 // set the generation members used by allocate_in_expanded_heap
29978 // and switch to ephemeral generation
29979 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29981 if (!should_promote_ephemeral)
29983 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29984 active_new_gen_number);
29989 repair_allocation_in_expanded_heap (consing_gen);
29992 // assert that the generation gap for all of the ephemeral generations were allocated.
29995 int generation_num = max_generation-1;
29996 while (generation_num >= 0)
29998 generation* gen = generation_of (generation_num);
29999 assert (generation_plan_allocation_start (gen));
30005 if (!new_segment_p)
30007 dprintf (2, ("Demoting ephemeral segment"));
30008 //demote the entire segment.
30009 settings.demotion = TRUE;
30010 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
30011 demotion_low = heap_segment_mem (ephemeral_heap_segment);
30012 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
30016 demotion_low = MAX_PTR;
30018 #ifndef MULTIPLE_HEAPS
30019 settings.demotion = FALSE;
30020 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
30021 #endif //!MULTIPLE_HEAPS
30023 ptrdiff_t eph_size1 = total_ephemeral_size;
30024 MAYBE_UNUSED_VAR(eph_size1);
30026 if (!should_promote_ephemeral && new_segment_p)
30028 assert (eph_size1 <= eph_size);
30031 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
30033 // This is to catch when we accidently delete a segment that has pins.
30034 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
30037 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
30039 dprintf(2,("---- End of Heap Expansion ----"));
30040 return consing_gen;
30043 void gc_heap::set_static_data()
30045 static_data* pause_mode_sdata = static_data_table[latency_level];
30046 for (int i = 0; i < NUMBERGENERATIONS; i++)
30048 dynamic_data* dd = dynamic_data_of (i);
30049 static_data* sdata = &pause_mode_sdata[i];
30052 dd->min_size = sdata->min_size;
30054 dprintf (GTC_LOG, ("PM: %d, gen%d: min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
30055 settings.pause_mode,i,
30056 dd->min_size, dd_max_size (dd),
30057 sdata->fragmentation_limit, (int)(sdata->fragmentation_burden_limit * 100)));
30061 // Initialize the values that are not const.
30062 void gc_heap::init_static_data()
30064 size_t gen0_min_size = get_gen0_min_size();
30066 size_t gen0_max_size =
30067 #ifdef MULTIPLE_HEAPS
30068 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
30069 #else //MULTIPLE_HEAPS
30070 (gc_can_use_concurrent ?
30072 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
30073 #endif //MULTIPLE_HEAPS
30075 if (heap_hard_limit)
30077 size_t gen0_max_size_seg = soh_segment_size / 4;
30078 dprintf (GTC_LOG, ("limit gen0 max %Id->%Id", gen0_max_size, gen0_max_size_seg));
30079 gen0_max_size = min (gen0_max_size, gen0_max_size_seg);
30082 size_t gen0_max_size_config = (size_t)GCConfig::GetGCGen0MaxBudget();
30084 if (gen0_max_size_config)
30086 gen0_max_size = min (gen0_max_size, gen0_max_size_config);
30089 gen0_max_size = Align (gen0_max_size);
30091 gen0_min_size = min (gen0_min_size, gen0_max_size);
30093 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
30094 size_t gen1_max_size = (size_t)
30095 #ifdef MULTIPLE_HEAPS
30096 max (6*1024*1024, Align(soh_segment_size/2));
30097 #else //MULTIPLE_HEAPS
30098 (gc_can_use_concurrent ?
30100 max (6*1024*1024, Align(soh_segment_size/2)));
30101 #endif //MULTIPLE_HEAPS
30103 dprintf (GTC_LOG, ("gen0 min: %Id, max: %Id, gen1 max: %Id",
30104 gen0_min_size, gen0_max_size, gen1_max_size));
30106 for (int i = latency_level_first; i <= latency_level_last; i++)
30108 static_data_table[i][0].min_size = gen0_min_size;
30109 static_data_table[i][0].max_size = gen0_max_size;
30110 static_data_table[i][1].max_size = gen1_max_size;
30114 bool gc_heap::init_dynamic_data()
30116 qpf = GCToOSInterface::QueryPerformanceFrequency();
30118 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
30122 for (int i = 0; i <= max_generation+1; i++)
30124 dynamic_data* dd = dynamic_data_of (i);
30126 dd->time_clock = now;
30127 dd->current_size = 0;
30128 dd->promoted_size = 0;
30129 dd->collection_count = 0;
30130 dd->new_allocation = dd->min_size;
30131 dd->gc_new_allocation = dd->new_allocation;
30132 dd->desired_allocation = dd->new_allocation;
30133 dd->fragmentation = 0;
30136 #ifdef GC_CONFIG_DRIVEN
30137 if (heap_number == 0)
30139 #endif //GC_CONFIG_DRIVEN
30144 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
30146 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
30147 return ((limit - limit*cst) / (1.0f - (cst * limit)));
30153 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
30154 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
30155 //value of the budget
30156 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
30157 size_t previous_desired_allocation, size_t collection_count)
30159 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
30161 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
30162 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
30165 size_t smoothing = 3; // exponential smoothing factor
30166 if (smoothing > collection_count)
30167 smoothing = collection_count;
30168 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
30170 UNREFERENCED_PARAMETER(collection_count);
30172 return new_allocation;
30175 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
30176 size_t out, int gen_number,
30179 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30181 if (dd_begin_data_size (dd) == 0)
30183 size_t new_allocation = dd_min_size (dd);
30184 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
30185 return new_allocation;
30190 size_t previous_desired_allocation = dd_desired_allocation (dd);
30191 size_t current_size = dd_current_size (dd);
30192 float max_limit = dd_max_limit (dd);
30193 float limit = dd_limit (dd);
30194 size_t min_gc_size = dd_min_size (dd);
30196 size_t max_size = dd_max_size (dd);
30197 size_t new_allocation = 0;
30198 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
30199 if (gen_number >= max_generation)
30201 size_t new_size = 0;
30203 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
30205 f = surv_to_growth (cst, limit, max_limit);
30206 size_t max_growth_size = (size_t)(max_size / f);
30207 if (current_size >= max_growth_size)
30209 new_size = max_size;
30213 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
30216 assert ((new_size >= current_size) || (new_size == max_size));
30218 if (gen_number == max_generation)
30220 new_allocation = max((new_size - current_size), min_gc_size);
30222 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30223 dd_desired_allocation (dd), dd_collection_count (dd));
30225 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
30227 //reducing allocation in case of fragmentation
30228 size_t new_allocation1 = max (min_gc_size,
30230 (size_t)((float)new_allocation * current_size /
30231 ((float)current_size + 2*dd_fragmentation (dd))));
30232 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
30233 new_allocation, new_allocation1));
30234 new_allocation = new_allocation1;
30237 else //large object heap
30239 uint32_t memory_load = 0;
30240 uint64_t available_physical = 0;
30241 get_memory_info (&memory_load, &available_physical);
30243 if (heap_hard_limit)
30245 size_t loh_allocated = 0;
30246 size_t loh_committed = committed_size (true, &loh_allocated);
30247 dprintf (1, ("GC#%Id h%d, GMI: LOH budget, LOH commit %Id (obj %Id, frag %Id), total commit: %Id (recorded: %Id)",
30248 (size_t)settings.gc_index, heap_number,
30249 loh_committed, loh_allocated,
30250 dd_fragmentation (dynamic_data_of (max_generation + 1)),
30251 get_total_committed_size(), (current_total_committed - current_total_committed_bookkeeping)));
30254 if (heap_number == 0)
30255 settings.exit_memory_load = memory_load;
30256 if (available_physical > 1024*1024)
30257 available_physical -= 1024*1024;
30259 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
30260 if (available_free > (uint64_t)MAX_PTR)
30262 available_free = (uint64_t)MAX_PTR;
30265 //try to avoid OOM during large object allocation
30266 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
30267 (size_t)available_free),
30268 max ((current_size/4), min_gc_size));
30270 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30271 dd_desired_allocation (dd), dd_collection_count (dd));
30277 size_t survivors = out;
30278 cst = float (survivors) / float (dd_begin_data_size (dd));
30279 f = surv_to_growth (cst, limit, max_limit);
30280 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
30282 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30283 dd_desired_allocation (dd), dd_collection_count (dd));
30285 if (gen_number == 0)
30290 //printf ("%f, %Id\n", cst, new_allocation);
30291 size_t free_space = generation_free_list_space (generation_of (gen_number));
30292 // DTREVIEW - is min_gc_size really a good choice?
30293 // on 64-bit this will almost always be true.
30294 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
30295 if (free_space > min_gc_size)
30297 settings.gen0_reduction_count = 2;
30301 if (settings.gen0_reduction_count > 0)
30302 settings.gen0_reduction_count--;
30305 if (settings.gen0_reduction_count > 0)
30307 dprintf (2, ("Reducing new allocation based on fragmentation"));
30308 new_allocation = min (new_allocation,
30309 max (min_gc_size, (max_size/3)));
30314 size_t new_allocation_ret =
30315 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
30316 int gen_data_index = gen_number;
30317 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
30318 gen_data->new_allocation = new_allocation_ret;
30320 dd_surv (dd) = cst;
30322 #ifdef SIMPLE_DPRINTF
30323 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
30324 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
30325 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30327 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
30328 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
30329 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
30330 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30331 #endif //SIMPLE_DPRINTF
30333 return new_allocation_ret;
30337 //returns the planned size of a generation (including free list element)
30338 size_t gc_heap::generation_plan_size (int gen_number)
30340 if (0 == gen_number)
30341 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
30342 generation_plan_allocation_start (generation_of (gen_number))),
30343 (int)Align (min_obj_size));
30346 generation* gen = generation_of (gen_number);
30347 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30348 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30349 generation_plan_allocation_start (generation_of (gen_number)));
30352 size_t gensize = 0;
30353 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30355 PREFIX_ASSUME(seg != NULL);
30357 while (seg && (seg != ephemeral_heap_segment))
30359 gensize += heap_segment_plan_allocated (seg) -
30360 heap_segment_mem (seg);
30361 seg = heap_segment_next_rw (seg);
30365 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30366 heap_segment_mem (ephemeral_heap_segment));
30374 //returns the size of a generation (including free list element)
30375 size_t gc_heap::generation_size (int gen_number)
30377 if (0 == gen_number)
30378 return max((heap_segment_allocated (ephemeral_heap_segment) -
30379 generation_allocation_start (generation_of (gen_number))),
30380 (int)Align (min_obj_size));
30383 generation* gen = generation_of (gen_number);
30384 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30385 return (generation_allocation_start (generation_of (gen_number - 1)) -
30386 generation_allocation_start (generation_of (gen_number)));
30389 size_t gensize = 0;
30390 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30392 PREFIX_ASSUME(seg != NULL);
30394 while (seg && (seg != ephemeral_heap_segment))
30396 gensize += heap_segment_allocated (seg) -
30397 heap_segment_mem (seg);
30398 seg = heap_segment_next_rw (seg);
30402 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
30403 heap_segment_mem (ephemeral_heap_segment));
30412 size_t gc_heap::compute_in (int gen_number)
30414 assert (gen_number != 0);
30415 dynamic_data* dd = dynamic_data_of (gen_number);
30417 size_t in = generation_allocation_size (generation_of (gen_number));
30419 if (gen_number == max_generation && ephemeral_promotion)
30422 for (int i = 0; i <= max_generation; i++)
30424 dynamic_data* dd = dynamic_data_of (i);
30425 in += dd_survived_size (dd);
30426 if (i != max_generation)
30428 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
30433 dd_gc_new_allocation (dd) -= in;
30434 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30436 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30437 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30440 generation_allocation_size (generation_of (gen_number)) = 0;
30444 void gc_heap::compute_promoted_allocation (int gen_number)
30446 compute_in (gen_number);
30451 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
30452 size_t total_new_allocation,
30453 size_t total_min_allocation)
30455 if (memory_load < MAX_ALLOWED_MEM_LOAD)
30457 // If the total of memory load and gen0 budget exceeds
30458 // our max memory load limit, trim the gen0 budget so the total
30459 // is the max memory load limit.
30460 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
30461 return min (total_new_allocation, remain_memory_load);
30465 return max (mem_one_percent, total_min_allocation);
30469 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
30471 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
30473 size_t final_new_allocation = new_allocation;
30474 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
30476 uint32_t num_heaps = 1;
30478 #ifdef MULTIPLE_HEAPS
30479 num_heaps = gc_heap::n_heaps;
30480 #endif //MULTIPLE_HEAPS
30482 size_t total_new_allocation = new_allocation * num_heaps;
30483 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
30485 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
30486 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
30488 uint32_t memory_load = 0;
30489 get_memory_info (&memory_load);
30490 settings.exit_memory_load = memory_load;
30491 dprintf (2, ("Current emory load: %d", memory_load));
30493 size_t final_total =
30494 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
30495 size_t max_new_allocation =
30496 #ifdef MULTIPLE_HEAPS
30497 dd_max_size (g_heaps[0]->dynamic_data_of (0));
30498 #else //MULTIPLE_HEAPS
30499 dd_max_size (dynamic_data_of (0));
30500 #endif //MULTIPLE_HEAPS
30502 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
30506 if (final_new_allocation < new_allocation)
30508 settings.gen0_reduction_count = 2;
30511 return final_new_allocation;
30516 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
30518 #ifdef BACKGROUND_GC
30519 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
30521 return &gc_data_per_heap;
30522 #endif //BACKGROUND_GC
30525 void gc_heap::compute_new_dynamic_data (int gen_number)
30527 PREFIX_ASSUME(gen_number >= 0);
30528 PREFIX_ASSUME(gen_number <= max_generation);
30530 dynamic_data* dd = dynamic_data_of (gen_number);
30531 generation* gen = generation_of (gen_number);
30532 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
30534 size_t total_gen_size = generation_size (gen_number);
30535 //keep track of fragmentation
30536 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
30537 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30539 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30541 size_t out = dd_survived_size (dd);
30543 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30544 gen_data->size_after = total_gen_size;
30545 gen_data->free_list_space_after = generation_free_list_space (gen);
30546 gen_data->free_obj_space_after = generation_free_obj_space (gen);
30548 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
30550 // When we are in the low latency mode, we can still be
30551 // condemning more than gen1's 'cause of induced GCs.
30552 dd_desired_allocation (dd) = low_latency_alloc;
30556 if (gen_number == 0)
30558 //compensate for dead finalizable objects promotion.
30559 //they shoudn't be counted for growth.
30560 size_t final_promoted = 0;
30561 final_promoted = min (promoted_bytes (heap_number), out);
30562 // Prefast: this is clear from above but prefast needs to be told explicitly
30563 PREFIX_ASSUME(final_promoted <= out);
30565 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
30566 dd_freach_previous_promotion (dd) = final_promoted;
30567 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
30569 if (settings.condemned_generation == 0)
30571 //there is no noise.
30572 dd_desired_allocation (dd) = lower_bound;
30576 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
30578 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
30579 //assert ( lower_bound <= higher_bound);
30581 //discount the noise. Change the desired allocation
30582 //only if the previous value is outside of the range.
30583 if (dd_desired_allocation (dd) < lower_bound)
30585 dd_desired_allocation (dd) = lower_bound;
30587 else if (dd_desired_allocation (dd) > higher_bound)
30589 dd_desired_allocation (dd) = higher_bound;
30591 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
30592 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
30593 #endif // BIT64 && !MULTIPLE_HEAPS
30594 trim_youngest_desired_low_memory();
30595 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
30600 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
30604 gen_data->pinned_surv = dd_pinned_survived_size (dd);
30605 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
30607 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30608 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30611 dd_promoted_size (dd) = out;
30612 if (gen_number == max_generation)
30614 dd = dynamic_data_of (max_generation+1);
30615 total_gen_size = generation_size (max_generation + 1);
30616 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30617 generation_free_obj_space (large_object_generation);
30618 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30619 dd_survived_size (dd) = dd_current_size (dd);
30621 out = dd_current_size (dd);
30622 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30623 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30624 get_alignment_constant (FALSE));
30625 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30627 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30628 gen_data->size_after = total_gen_size;
30629 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30630 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30631 gen_data->npinned_surv = out;
30632 #ifdef BACKGROUND_GC
30633 end_loh_size = total_gen_size;
30634 #endif //BACKGROUND_GC
30636 dd_promoted_size (dd) = out;
30640 void gc_heap::trim_youngest_desired_low_memory()
30642 if (g_low_memory_status)
30644 size_t committed_mem = 0;
30645 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30648 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30649 seg = heap_segment_next (seg);
30651 seg = generation_start_segment (generation_of (max_generation + 1));
30654 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30655 seg = heap_segment_next (seg);
30658 dynamic_data* dd = dynamic_data_of (0);
30659 size_t current = dd_desired_allocation (dd);
30660 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30662 dd_desired_allocation (dd) = min (current, candidate);
30666 void gc_heap::decommit_ephemeral_segment_pages()
30668 if (settings.concurrent)
30673 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30675 dynamic_data* dd = dynamic_data_of (0);
30677 #ifndef MULTIPLE_HEAPS
30678 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30679 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30680 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30682 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30684 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30687 if (ephemeral_elapsed >= decommit_timeout)
30689 slack_space = min (slack_space, gc_gen0_desired_high);
30691 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30692 gc_gen0_desired_high = 0;
30694 #endif //!MULTIPLE_HEAPS
30696 if (settings.condemned_generation >= (max_generation-1))
30698 size_t new_slack_space =
30700 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30702 #ifdef FEATURE_CORECLR
30703 dd_desired_allocation (dd);
30706 #endif //FEATURE_CORECLR
30709 slack_space = min (slack_space, new_slack_space);
30712 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30714 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30715 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30718 //This is meant to be called by decide_on_compacting.
30720 size_t gc_heap::generation_fragmentation (generation* gen,
30721 generation* consing_gen,
30725 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30726 // If the allocation pointer has reached the ephemeral segment
30727 // fine, otherwise the whole ephemeral segment is considered
30729 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30731 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30732 frag = end - alloc;
30735 // case when no survivors, allocated set to beginning
30738 dprintf (3, ("ephemeral frag: %Id", frag));
30741 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30742 heap_segment_mem (ephemeral_heap_segment));
30743 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30745 PREFIX_ASSUME(seg != NULL);
30747 while (seg != ephemeral_heap_segment)
30749 frag += (heap_segment_allocated (seg) -
30750 heap_segment_plan_allocated (seg));
30751 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30752 (heap_segment_allocated (seg) -
30753 heap_segment_plan_allocated (seg))));
30755 seg = heap_segment_next_rw (seg);
30758 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30759 //add the length of the dequeued plug free space
30761 while (bos < mark_stack_bos)
30763 frag += (pinned_len (pinned_plug_of (bos)));
30770 // for SOH this returns the total sizes of the generation and its
30771 // younger generation(s).
30772 // for LOH this returns just LOH size.
30773 size_t gc_heap::generation_sizes (generation* gen)
30776 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30777 result = (heap_segment_allocated (ephemeral_heap_segment) -
30778 generation_allocation_start (gen));
30781 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30783 PREFIX_ASSUME(seg != NULL);
30787 result += (heap_segment_allocated (seg) -
30788 heap_segment_mem (seg));
30789 seg = heap_segment_next_in_range (seg);
30796 size_t gc_heap::estimated_reclaim (int gen_number)
30798 dynamic_data* dd = dynamic_data_of (gen_number);
30799 size_t gen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
30800 size_t gen_total_size = gen_allocated + dd_current_size (dd);
30801 size_t est_gen_surv = (size_t)((float) (gen_total_size) * dd_surv (dd));
30802 size_t est_gen_free = gen_total_size - est_gen_surv + dd_fragmentation (dd);
30804 dprintf (GTC_LOG, ("h%d gen%d total size: %Id, est dead space: %Id (s: %d, allocated: %Id), frag: %Id",
30805 heap_number, gen_number,
30808 (int)(dd_surv (dd) * 100),
30810 dd_fragmentation (dd)));
30812 return est_gen_free;
30815 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30816 size_t fragmentation,
30817 BOOL& should_expand)
30819 BOOL should_compact = FALSE;
30820 should_expand = FALSE;
30821 generation* gen = generation_of (condemned_gen_number);
30822 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30823 size_t gen_sizes = generation_sizes(gen);
30824 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30825 (float (fragmentation) / gen_sizes) );
30827 dprintf (GTC_LOG, ("h%d g%d fragmentation: %Id (%d%%)",
30828 heap_number, settings.condemned_generation,
30829 fragmentation, (int)(fragmentation_burden * 100.0)));
30832 // for pure GC stress runs we need compaction, for GC stress "mix"
30833 // we need to ensure a better mix of compacting and sweeping collections
30834 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30835 && !g_pConfig->IsGCStressMix())
30836 should_compact = TRUE;
30839 // in GC stress "mix" mode, for stress induced collections make sure we
30840 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30841 // against the GC's determination, as it may lead to premature OOMs.
30842 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30844 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30845 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30846 if (compactions < sweeps / 10)
30848 should_compact = TRUE;
30852 #endif //STRESS_HEAP
30854 if (GCConfig::GetForceCompact())
30855 should_compact = TRUE;
30857 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30859 should_compact = TRUE;
30860 last_gc_before_oom = FALSE;
30861 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30864 if (settings.reason == reason_induced_compacting)
30866 dprintf (2, ("induced compacting GC"));
30867 should_compact = TRUE;
30868 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30871 if (settings.reason == reason_pm_full_gc)
30873 assert (condemned_gen_number == max_generation);
30874 if (heap_number == 0)
30876 dprintf (GTC_LOG, ("PM doing compacting full GC after a gen1"));
30878 should_compact = TRUE;
30881 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30882 fragmentation, (int) (100*fragmentation_burden)));
30884 if (provisional_mode_triggered && (condemned_gen_number == (max_generation - 1)))
30886 dprintf (GTC_LOG, ("gen1 in PM always compact"));
30887 should_compact = TRUE;
30890 if (!should_compact)
30892 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30894 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30895 should_compact = TRUE;
30896 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30900 if (should_compact)
30902 if ((condemned_gen_number >= (max_generation - 1)))
30904 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30906 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30907 should_expand = TRUE;
30913 BOOL high_memory = FALSE;
30916 if (!should_compact)
30918 // We are not putting this in dt_high_frag_p because it's not exactly
30919 // high fragmentation - it's just enough planned fragmentation for us to
30920 // want to compact. Also the "fragmentation" we are talking about here
30921 // is different from anywhere else.
30922 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30923 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30927 #ifdef BACKGROUND_GC
30928 // do not force compaction if this was a stress-induced GC
30929 IN_STRESS_HEAP(if (!settings.stress_induced))
30931 #endif // BACKGROUND_GC
30932 assert (settings.concurrent == FALSE);
30933 should_compact = TRUE;
30934 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30935 #ifdef BACKGROUND_GC
30937 #endif // BACKGROUND_GC
30941 // check for high memory situation
30942 if(!should_compact)
30944 uint32_t num_heaps = 1;
30945 #ifdef MULTIPLE_HEAPS
30946 num_heaps = gc_heap::n_heaps;
30947 #endif // MULTIPLE_HEAPS
30949 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30951 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30953 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30955 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30956 should_compact = TRUE;
30957 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30959 high_memory = TRUE;
30961 else if(settings.entry_memory_load >= v_high_memory_load_th)
30963 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30965 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30966 should_compact = TRUE;
30967 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30969 high_memory = TRUE;
30975 // The purpose of calling ensure_gap_allocation here is to make sure
30976 // that we actually are able to commit the memory to allocate generation
30978 if ((should_compact == FALSE) &&
30979 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30981 should_compact = TRUE;
30982 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30985 if (settings.condemned_generation == max_generation)
30987 //check the progress
30990 (high_memory && !should_compact) ||
30992 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30993 generation_allocation_start (generation_of (max_generation - 1))))
30995 dprintf (1, ("gen1 start %Ix->%Ix, gen2 size %Id->%Id, lock elevation",
30996 generation_allocation_start (generation_of (max_generation - 1)),
30997 generation_plan_allocation_start (generation_of (max_generation - 1)),
30998 generation_size (max_generation),
30999 generation_plan_size (max_generation)));
31000 //no progress -> lock
31001 settings.should_lock_elevation = TRUE;
31005 if (settings.pause_mode == pause_no_gc)
31007 should_compact = TRUE;
31008 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
31009 < soh_allocation_no_gc)
31011 should_expand = TRUE;
31015 dprintf (2, ("will %s(%s)", (should_compact ? "compact" : "sweep"), (should_expand ? "ex" : "")));
31016 return should_compact;
31019 size_t align_lower_good_size_allocation (size_t size)
31021 return (size/64)*64;
31024 size_t gc_heap::approximate_new_allocation()
31026 dynamic_data* dd0 = dynamic_data_of (0);
31027 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
31030 BOOL gc_heap::sufficient_space_end_seg (uint8_t* start, uint8_t* seg_end, size_t end_space_required, gc_tuning_point tp)
31032 BOOL can_fit = FALSE;
31033 size_t end_seg_space = (size_t)(seg_end - start);
31034 if (end_seg_space > end_space_required)
31036 // If hard limit is specified, and if we attributed all that's left in commit to the ephemeral seg
31037 // so we treat that as segment end, do we have enough space.
31038 if (heap_hard_limit)
31040 size_t left_in_commit = heap_hard_limit - current_total_committed;
31042 #ifdef MULTIPLE_HEAPS
31043 num_heaps = n_heaps;
31044 #endif //MULTIPLE_HEAPS
31045 left_in_commit /= num_heaps;
31046 if (left_in_commit > end_space_required)
31051 dprintf (2, ("h%d end seg %Id, but only %Id left in HARD LIMIT commit, required: %Id %s on eph (%d)",
31052 heap_number, end_seg_space,
31053 left_in_commit, end_space_required,
31054 (can_fit ? "ok" : "short"), (int)tp));
31063 // After we did a GC we expect to have at least this
31064 // much space at the end of the segment to satisfy
31065 // a reasonable amount of allocation requests.
31066 size_t gc_heap::end_space_after_gc()
31068 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
31071 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
31073 uint8_t* start = 0;
31075 if ((tp == tuning_deciding_condemned_gen) ||
31076 (tp == tuning_deciding_compaction))
31078 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
31079 if (settings.concurrent)
31081 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
31082 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31086 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
31087 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
31090 else if (tp == tuning_deciding_expansion)
31092 start = heap_segment_plan_allocated (ephemeral_heap_segment);
31093 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
31094 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
31098 assert (tp == tuning_deciding_full_gc);
31099 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
31100 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31101 start = alloc_allocated;
31104 if (start == 0) // empty ephemeral generations
31106 assert (tp == tuning_deciding_expansion);
31107 // if there are no survivors in the ephemeral segment,
31108 // this should be the beginning of ephemeral segment.
31109 start = generation_allocation_pointer (generation_of (max_generation));
31110 assert (start == heap_segment_mem (ephemeral_heap_segment));
31113 if (tp == tuning_deciding_expansion)
31115 assert (settings.condemned_generation >= (max_generation-1));
31116 size_t gen0size = approximate_new_allocation();
31117 size_t eph_size = gen0size;
31118 size_t gen_min_sizes = 0;
31120 for (int j = 1; j <= max_generation-1; j++)
31122 gen_min_sizes += 2*dd_min_size (dynamic_data_of(j));
31125 eph_size += gen_min_sizes;
31127 dprintf (3, ("h%d deciding on expansion, need %Id (gen0: %Id, 2*min: %Id)",
31128 heap_number, gen0size, gen_min_sizes, eph_size));
31130 // We must find room for one large object and enough room for gen0size
31131 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
31133 dprintf (3, ("Enough room before end of segment"));
31138 size_t room = align_lower_good_size_allocation
31139 (heap_segment_reserved (ephemeral_heap_segment) - start);
31140 size_t end_seg = room;
31142 //look at the plug free space
31143 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
31144 bool large_chunk_found = FALSE;
31146 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
31147 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
31148 if (gen0start == 0)
31150 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
31152 while ((bos < mark_stack_bos) &&
31153 !((room >= gen0size) && large_chunk_found))
31155 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
31156 if (in_range_for_segment (plug, ephemeral_heap_segment))
31158 if (plug >= gen0start)
31160 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
31162 if (!large_chunk_found)
31164 large_chunk_found = (chunk >= largest_alloc);
31166 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
31167 room, large_chunk_found));
31173 if (room >= gen0size)
31175 if (large_chunk_found)
31177 sufficient_gen0_space_p = TRUE;
31179 dprintf (3, ("Enough room"));
31184 // now we need to find largest_alloc at the end of the segment.
31185 if (end_seg >= end_space_after_gc())
31187 dprintf (3, ("Enough room (may need end of seg)"));
31193 dprintf (3, ("Not enough room"));
31199 size_t end_space = 0;
31200 dynamic_data* dd = dynamic_data_of (0);
31201 if ((tp == tuning_deciding_condemned_gen) ||
31202 (tp == tuning_deciding_full_gc))
31204 end_space = max (2*dd_min_size (dd), end_space_after_gc());
31208 assert (tp == tuning_deciding_compaction);
31209 end_space = approximate_new_allocation();
31212 BOOL can_fit = sufficient_space_end_seg (start, heap_segment_reserved (ephemeral_heap_segment), end_space, tp);
31218 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
31220 //create a new alloc context because gen3context is shared.
31221 alloc_context acontext;
31222 acontext.alloc_ptr = 0;
31223 acontext.alloc_limit = 0;
31224 acontext.alloc_bytes = 0;
31225 #ifdef MULTIPLE_HEAPS
31226 acontext.set_alloc_heap(vm_heap);
31227 #endif //MULTIPLE_HEAPS
31230 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
31232 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
31235 if (jsize >= maxObjectSize)
31237 if (GCConfig::GetBreakOnOOM())
31239 GCToOSInterface::DebugBreak();
31244 size_t size = AlignQword (jsize);
31245 int align_const = get_alignment_constant (FALSE);
31246 #ifdef FEATURE_LOH_COMPACTION
31247 size_t pad = Align (loh_padding_obj_size, align_const);
31250 #endif //FEATURE_LOH_COMPACTION
31252 assert (size >= Align (min_obj_size, align_const));
31254 #pragma inline_depth(0)
31256 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
31262 #pragma inline_depth(20)
31266 uint8_t* current_lowest_address = lowest_address;
31267 uint8_t* current_highest_address = highest_address;
31268 #ifdef BACKGROUND_GC
31269 if (recursive_gc_sync::background_running_p())
31271 current_lowest_address = background_saved_lowest_address;
31272 current_highest_address = background_saved_highest_address;
31274 #endif //BACKGROUND_GC
31275 #endif // MARK_ARRAY
31277 #ifdef FEATURE_LOH_COMPACTION
31278 // The GC allocator made a free object already in this alloc context and
31279 // adjusted the alloc_ptr accordingly.
31280 #endif //FEATURE_LOH_COMPACTION
31282 uint8_t* result = acontext.alloc_ptr;
31284 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
31285 alloc_bytes += size;
31287 CObjectHeader* obj = (CObjectHeader*)result;
31290 if (recursive_gc_sync::background_running_p())
31292 if ((result < current_highest_address) && (result >= current_lowest_address))
31294 dprintf (3, ("Clearing mark bit at address %Ix",
31295 (size_t)(&mark_array [mark_word_of (result)])));
31297 mark_array_clear_marked (result);
31299 #ifdef BACKGROUND_GC
31300 //the object has to cover one full mark uint32_t
31301 assert (size > mark_word_size);
31302 if (current_c_gc_state != c_gc_state_free)
31304 dprintf (3, ("Concurrent allocation of a large object %Ix",
31306 //mark the new block specially so we know it is a new object
31307 if ((result < current_highest_address) && (result >= current_lowest_address))
31309 dprintf (3, ("Setting mark bit at address %Ix",
31310 (size_t)(&mark_array [mark_word_of (result)])));
31312 mark_array_set_marked (result);
31315 #endif //BACKGROUND_GC
31317 #endif //MARK_ARRAY
31320 assert ((size_t)obj == Align ((size_t)obj, align_const));
31325 void reset_memory (uint8_t* o, size_t sizeo)
31327 if (sizeo > 128 * 1024)
31329 // We cannot reset the memory for the useful part of a free object.
31330 size_t size_to_skip = min_free_list - plug_skew;
31332 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
31333 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
31334 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
31335 // on write watched memory.
31338 #ifdef MULTIPLE_HEAPS
31339 bool unlock_p = true;
31341 // We don't do unlock because there could be many processes using workstation GC and it's
31342 // bad perf to have many threads doing unlock at the same time.
31343 bool unlock_p = false;
31344 #endif //MULTIPLE_HEAPS
31346 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
31351 void gc_heap::reset_large_object (uint8_t* o)
31353 // If it's a large object, allow the O/S to discard the backing store for these pages.
31354 reset_memory (o, size(o));
31357 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
31360 // It shouldn't be necessary to do these comparisons because this is only used for blocking
31361 // GCs and LOH segments cannot be out of range.
31362 if ((o >= lowest_address) && (o < highest_address))
31382 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
31384 // Now walk the portion of memory that is actually being relocated.
31385 walk_relocation (profiling_context, fn);
31387 #ifdef FEATURE_LOH_COMPACTION
31388 if (loh_compacted_p)
31390 walk_relocation_for_loh (profiling_context, fn);
31392 #endif //FEATURE_LOH_COMPACTION
31395 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
31397 generation* gen = large_object_generation;
31398 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
31400 PREFIX_ASSUME(seg != NULL);
31402 uint8_t* o = generation_allocation_start (gen);
31403 uint8_t* plug_end = o;
31404 uint8_t* plug_start = o;
31408 if (o >= heap_segment_allocated (seg))
31410 seg = heap_segment_next (seg);
31414 o = heap_segment_mem (seg);
31416 if (large_object_marked(o, FALSE))
31423 o = o + AlignQword (size (o));
31424 if (o >= heap_segment_allocated (seg))
31428 m = large_object_marked (o, FALSE);
31433 fn (plug_start, plug_end, 0, profiling_context, false, false);
31437 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31439 o = o + AlignQword (size (o));
31445 #ifdef BACKGROUND_GC
31447 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
31450 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
31452 if (mark_array_marked (o))
31456 mark_array_clear_marked (o);
31457 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
31458 dprintf (3, ("CM: %Ix", o));
31468 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
31472 void gc_heap::background_delay_delete_loh_segments()
31474 generation* gen = large_object_generation;
31475 heap_segment* seg = heap_segment_rw (generation_start_segment (large_object_generation));
31476 heap_segment* prev_seg = 0;
31480 heap_segment* next_seg = heap_segment_next (seg);
31481 if (seg->flags & heap_segment_flags_loh_delete)
31483 dprintf (3, ("deleting %Ix-%Ix-%Ix", (size_t)seg, heap_segment_allocated (seg), heap_segment_reserved (seg)));
31484 delete_heap_segment (seg, (GCConfig::GetRetainVM() != 0));
31485 heap_segment_next (prev_seg) = next_seg;
31496 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
31499 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
31502 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
31507 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
31508 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
31510 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
31511 memset (start, b, (end - start));
31514 #endif //VERIFY_HEAP
31517 void gc_heap::generation_delete_heap_segment (generation* gen,
31519 heap_segment* prev_seg,
31520 heap_segment* next_seg)
31522 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
31523 if (gen == large_object_generation)
31525 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
31527 // We cannot thread segs in here onto freeable_large_heap_segment because
31528 // grow_brick_card_tables could be committing mark array which needs to read
31529 // the seg list. So we delay it till next time we suspend EE.
31530 seg->flags |= heap_segment_flags_loh_delete;
31531 // Since we will be decommitting the seg, we need to prevent heap verification
31532 // to verify this segment.
31533 heap_segment_allocated (seg) = heap_segment_mem (seg);
31537 if (seg == ephemeral_heap_segment)
31542 heap_segment_next (next_seg) = prev_seg;
31544 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
31545 heap_segment_next (seg) = freeable_small_heap_segment;
31546 freeable_small_heap_segment = seg;
31549 decommit_heap_segment (seg);
31550 seg->flags |= heap_segment_flags_decommitted;
31552 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31555 void gc_heap::process_background_segment_end (heap_segment* seg,
31557 uint8_t* last_plug_end,
31558 heap_segment* start_seg,
31562 uint8_t* allocated = heap_segment_allocated (seg);
31563 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31564 BOOL loh_p = heap_segment_loh_p (seg);
31566 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
31567 (size_t)heap_segment_mem (seg), background_allocated, allocated));
31569 if (!loh_p && (allocated != background_allocated))
31571 assert (gen != large_object_generation);
31573 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
31574 (size_t)last_plug_end, background_allocated));
31575 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
31578 fix_brick_to_highest (last_plug_end, background_allocated);
31580 // When we allowed fgc's during going through gaps, we could have erased the brick
31581 // that corresponds to bgc_allocated 'cause we had to update the brick there,
31582 // recover it here.
31583 fix_brick_to_highest (background_allocated, background_allocated);
31587 // by default, if allocated == background_allocated, it can't
31588 // be the ephemeral segment.
31589 if (seg == ephemeral_heap_segment)
31594 if (allocated == heap_segment_mem (seg))
31596 // this can happen with LOH segments when multiple threads
31597 // allocate new segments and not all of them were needed to
31598 // satisfy allocation requests.
31599 assert (gen == large_object_generation);
31602 if (last_plug_end == heap_segment_mem (seg))
31604 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
31605 (size_t)allocated, (*delete_p ? "should" : "should not")));
31607 if (seg != start_seg)
31614 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
31615 heap_segment_allocated (seg) = last_plug_end;
31616 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31618 decommit_heap_segment_pages (seg, 0);
31622 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
31623 bgc_verify_mark_array_cleared (seg);
31626 void gc_heap::process_n_background_segments (heap_segment* seg,
31627 heap_segment* prev_seg,
31630 assert (gen != large_object_generation);
31634 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
31635 heap_segment* next_seg = heap_segment_next (seg);
31637 if (heap_segment_read_only_p (seg))
31643 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
31645 // This can happen - if we have a LOH segment where nothing survived
31646 // or a SOH segment allocated by a gen1 GC when BGC was going where
31647 // nothing survived last time we did a gen1 GC.
31648 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31656 verify_soh_segment_list();
31662 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
31664 BOOL consider_bgc_mark_p,
31665 BOOL check_current_sweep_p,
31666 BOOL check_saved_sweep_p)
31668 // the logic for this function must be kept in sync with the analogous function
31669 // in ToolBox\SOS\Strike\gc.cpp
31671 // TRUE means we don't need to check the bgc mark bit
31672 // FALSE means we do.
31673 BOOL no_bgc_mark_p = FALSE;
31675 if (consider_bgc_mark_p)
31677 if (check_current_sweep_p && (o < current_sweep_pos))
31679 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
31680 no_bgc_mark_p = TRUE;
31683 if (!no_bgc_mark_p)
31685 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
31687 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
31688 no_bgc_mark_p = TRUE;
31691 if (!check_saved_sweep_p)
31693 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31694 // if this was the saved ephemeral segment, check_saved_sweep_p
31695 // would've been true.
31696 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
31697 // background_allocated could be 0 for the new segments acquired during bgc
31698 // sweep and we still want no_bgc_mark_p to be true.
31699 if (o >= background_allocated)
31701 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
31702 no_bgc_mark_p = TRUE;
31709 no_bgc_mark_p = TRUE;
31712 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31713 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31716 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31717 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31718 // current sweep position or not.
31719 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31720 BOOL* consider_bgc_mark_p,
31721 BOOL* check_current_sweep_p,
31722 BOOL* check_saved_sweep_p)
31724 // the logic for this function must be kept in sync with the analogous function
31725 // in ToolBox\SOS\Strike\gc.cpp
31726 *consider_bgc_mark_p = FALSE;
31727 *check_current_sweep_p = FALSE;
31728 *check_saved_sweep_p = FALSE;
31730 if (current_c_gc_state == c_gc_state_planning)
31732 // We are doing the current_sweep_pos comparison here because we have yet to
31733 // turn on the swept flag for the segment but in_range_for_segment will return
31734 // FALSE if the address is the same as reserved.
31735 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31737 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31741 *consider_bgc_mark_p = TRUE;
31743 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31745 if (seg == saved_sweep_ephemeral_seg)
31747 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31748 *check_saved_sweep_p = TRUE;
31751 if (in_range_for_segment (current_sweep_pos, seg))
31753 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31754 current_sweep_pos, seg));
31755 *check_current_sweep_p = TRUE;
31761 void gc_heap::background_ephemeral_sweep()
31763 dprintf (3, ("bgc ephemeral sweep"));
31765 int align_const = get_alignment_constant (TRUE);
31767 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31768 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31770 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31771 // we thread onto a list first then publish it when we are done.
31772 allocator youngest_free_list;
31773 size_t youngest_free_list_space = 0;
31774 size_t youngest_free_obj_space = 0;
31776 youngest_free_list.clear();
31778 for (int i = 0; i <= (max_generation - 1); i++)
31780 generation* gen_to_reset = generation_of (i);
31781 assert (generation_free_list_space (gen_to_reset) == 0);
31782 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31783 // something there.
31786 for (int i = (max_generation - 1); i >= 0; i--)
31788 generation* current_gen = generation_of (i);
31789 uint8_t* o = generation_allocation_start (current_gen);
31790 //Skip the generation gap object
31791 o = o + Align(size (o), align_const);
31792 uint8_t* end = ((i > 0) ?
31793 generation_allocation_start (generation_of (i - 1)) :
31794 heap_segment_allocated (ephemeral_heap_segment));
31796 uint8_t* plug_end = o;
31797 uint8_t* plug_start = o;
31798 BOOL marked_p = FALSE;
31802 marked_p = background_object_marked (o, TRUE);
31806 size_t plug_size = plug_start - plug_end;
31810 thread_gap (plug_end, plug_size, current_gen);
31816 make_unused_array (plug_end, plug_size);
31817 if (plug_size >= min_free_list)
31819 youngest_free_list_space += plug_size;
31820 youngest_free_list.thread_item (plug_end, plug_size);
31824 youngest_free_obj_space += plug_size;
31829 fix_brick_to_highest (plug_end, plug_start);
31830 fix_brick_to_highest (plug_start, plug_start);
31835 o = o + Align (size (o), align_const);
31841 m = background_object_marked (o, TRUE);
31844 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31848 while ((o < end) && !background_object_marked (o, FALSE))
31850 o = o + Align (size (o), align_const);
31855 if (plug_end != end)
31859 thread_gap (plug_end, end - plug_end, current_gen);
31860 fix_brick_to_highest (plug_end, end);
31864 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31865 // the following line is temporary.
31866 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31868 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31870 make_unused_array (plug_end, (end - plug_end));
31872 #endif //VERIFY_HEAP
31876 dd_fragmentation (dynamic_data_of (i)) =
31877 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31880 generation* youngest_gen = generation_of (0);
31881 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31882 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31883 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31884 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31887 void gc_heap::background_sweep()
31889 generation* gen = generation_of (max_generation);
31890 dynamic_data* dd = dynamic_data_of (max_generation);
31891 // For SOH segments we go backwards.
31892 heap_segment* start_seg = ephemeral_heap_segment;
31893 PREFIX_ASSUME(start_seg != NULL);
31894 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31895 heap_segment* seg = start_seg;
31896 uint8_t* o = heap_segment_mem (seg);
31898 heap_segment* prev_seg = heap_segment_next (seg);
31899 int align_const = get_alignment_constant (TRUE);
31902 assert (o == generation_allocation_start (generation_of (max_generation)));
31903 o = o + Align(size (o), align_const);
31906 uint8_t* plug_end = o;
31907 uint8_t* plug_start = o;
31908 next_sweep_obj = o;
31909 current_sweep_pos = o;
31911 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31912 uint8_t* end = heap_segment_background_allocated (seg);
31913 BOOL delete_p = FALSE;
31915 //concurrent_print_time_delta ("finished with mark and start with sweep");
31916 concurrent_print_time_delta ("Sw");
31917 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31919 //block concurrent allocation for large objects
31920 dprintf (3, ("lh state: planning"));
31921 if (gc_lh_block_event.IsValid())
31923 gc_lh_block_event.Reset();
31926 for (int i = 0; i <= (max_generation + 1); i++)
31928 generation* gen_to_reset = generation_of (i);
31929 generation_allocator (gen_to_reset)->clear();
31930 generation_free_list_space (gen_to_reset) = 0;
31931 generation_free_obj_space (gen_to_reset) = 0;
31932 generation_free_list_allocated (gen_to_reset) = 0;
31933 generation_end_seg_allocated (gen_to_reset) = 0;
31934 generation_condemned_allocated (gen_to_reset) = 0;
31935 //reset the allocation so foreground gc can allocate into older generation
31936 generation_allocation_pointer (gen_to_reset)= 0;
31937 generation_allocation_limit (gen_to_reset) = 0;
31938 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31941 FIRE_EVENT(BGC2ndNonConEnd);
31943 loh_alloc_thread_count = 0;
31944 current_bgc_state = bgc_sweep_soh;
31945 verify_soh_segment_list();
31947 #ifdef FEATURE_BASICFREEZE
31948 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31949 ro_segments_in_range)
31951 sweep_ro_segments (generation_start_segment (gen));
31953 #endif // FEATURE_BASICFREEZE
31955 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31956 if (current_c_gc_state != c_gc_state_planning)
31958 current_c_gc_state = c_gc_state_planning;
31961 concurrent_print_time_delta ("Swe");
31963 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31964 PREFIX_ASSUME(loh_seg != NULL);
31967 loh_seg->flags &= ~heap_segment_flags_swept;
31968 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31969 loh_seg = heap_segment_next_rw (loh_seg);
31972 #ifdef MULTIPLE_HEAPS
31973 bgc_t_join.join(this, gc_join_restart_ee);
31974 if (bgc_t_join.joined())
31975 #endif //MULTIPLE_HEAPS
31977 #ifdef MULTIPLE_HEAPS
31978 dprintf(2, ("Starting BGC threads for resuming EE"));
31979 bgc_t_join.restart();
31980 #endif //MULTIPLE_HEAPS
31983 if (heap_number == 0)
31988 FIRE_EVENT(BGC2ndConBegin);
31990 background_ephemeral_sweep();
31992 concurrent_print_time_delta ("Swe eph");
31994 #ifdef MULTIPLE_HEAPS
31995 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31996 if (bgc_t_join.joined())
31997 #endif //MULTIPLE_HEAPS
31999 #ifdef FEATURE_EVENT_TRACE
32000 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
32001 GCEventKeyword_GCHeapSurvivalAndMovement,
32002 GCEventLevel_Information);
32003 #endif //FEATURE_EVENT_TRACE
32005 leave_spin_lock (&gc_lock);
32007 #ifdef MULTIPLE_HEAPS
32008 dprintf(2, ("Starting BGC threads for BGC sweeping"));
32009 bgc_t_join.restart();
32010 #endif //MULTIPLE_HEAPS
32013 disable_preemptive (true);
32015 dprintf (2, ("bgs: sweeping gen2 objects"));
32016 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32017 (size_t)heap_segment_mem (seg),
32018 (size_t)heap_segment_allocated (seg),
32019 (size_t)heap_segment_background_allocated (seg)));
32021 int num_objs = 256;
32022 int current_num_objs = 0;
32023 heap_segment* next_seg = 0;
32029 if (gen == large_object_generation)
32031 next_seg = heap_segment_next (seg);
32035 next_seg = heap_segment_prev (fseg, seg);
32040 if (!heap_segment_read_only_p (seg))
32042 if (gen == large_object_generation)
32044 // we can treat all LOH segments as in the bgc domain
32045 // regardless of whether we saw in bgc mark or not
32046 // because we don't allow LOH allocations during bgc
32047 // sweep anyway - the LOH segments can't change.
32048 process_background_segment_end (seg, gen, plug_end,
32049 start_seg, &delete_p);
32053 assert (heap_segment_background_allocated (seg) != 0);
32054 process_background_segment_end (seg, gen, plug_end,
32055 start_seg, &delete_p);
32057 assert (next_seg || !delete_p);
32063 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
32068 dprintf (2, ("seg %Ix has been swept", seg));
32069 seg->flags |= heap_segment_flags_swept;
32072 verify_soh_segment_list();
32076 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
32080 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32082 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32084 if (gen != large_object_generation)
32086 dprintf (2, ("bgs: sweeping gen3 objects"));
32087 concurrent_print_time_delta ("Swe SOH");
32088 FIRE_EVENT(BGC1stSweepEnd, 0);
32090 enter_spin_lock (&more_space_lock_loh);
32091 add_saved_spinlock_info (true, me_acquire, mt_bgc_loh_sweep);
32093 concurrent_print_time_delta ("Swe LOH took msl");
32095 // We wait till all allocating threads are completely done.
32096 int spin_count = yp_spin_count_unit;
32097 while (loh_alloc_thread_count)
32099 spin_and_switch (spin_count, (loh_alloc_thread_count == 0));
32102 current_bgc_state = bgc_sweep_loh;
32103 gen = generation_of (max_generation+1);
32104 start_seg = heap_segment_rw (generation_start_segment (gen));
32106 PREFIX_ASSUME(start_seg != NULL);
32110 o = generation_allocation_start (gen);
32111 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
32112 align_const = get_alignment_constant (FALSE);
32113 o = o + Align(size (o), align_const);
32115 end = heap_segment_allocated (seg);
32116 dprintf (2, ("sweeping gen3 objects"));
32117 generation_free_obj_space (gen) = 0;
32118 generation_allocator (gen)->clear();
32119 generation_free_list_space (gen) = 0;
32121 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32122 (size_t)heap_segment_mem (seg),
32123 (size_t)heap_segment_allocated (seg),
32124 (size_t)heap_segment_background_allocated (seg)));
32131 o = heap_segment_mem (seg);
32134 assert (gen != large_object_generation);
32135 assert (o == generation_allocation_start (generation_of (max_generation)));
32136 align_const = get_alignment_constant (TRUE);
32137 o = o + Align(size (o), align_const);
32141 current_sweep_pos = o;
32142 next_sweep_obj = o;
32145 end = background_next_end (seg, (gen == large_object_generation));
32146 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32147 (size_t)heap_segment_mem (seg),
32148 (size_t)heap_segment_allocated (seg),
32149 (size_t)heap_segment_background_allocated (seg)));
32153 if ((o < end) && background_object_marked (o, TRUE))
32156 if (gen == large_object_generation)
32158 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
32161 thread_gap (plug_end, plug_start-plug_end, gen);
32162 if (gen != large_object_generation)
32164 add_gen_free (max_generation, plug_start-plug_end);
32165 fix_brick_to_highest (plug_end, plug_start);
32166 // we need to fix the brick for the next plug here 'cause an FGC can
32167 // happen and can't read a stale brick.
32168 fix_brick_to_highest (plug_start, plug_start);
32175 next_sweep_obj = o + Align(size (o), align_const);
32176 current_num_objs++;
32177 if (current_num_objs >= num_objs)
32179 current_sweep_pos = next_sweep_obj;
32182 current_num_objs = 0;
32185 o = next_sweep_obj;
32191 m = background_object_marked (o, TRUE);
32194 if (gen != large_object_generation)
32196 add_gen_plug (max_generation, plug_end-plug_start);
32197 dd_survived_size (dd) += (plug_end - plug_start);
32199 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32203 while ((o < end) && !background_object_marked (o, FALSE))
32205 next_sweep_obj = o + Align(size (o), align_const);;
32206 current_num_objs++;
32207 if (current_num_objs >= num_objs)
32209 current_sweep_pos = plug_end;
32210 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
32212 current_num_objs = 0;
32215 o = next_sweep_obj;
32220 size_t total_loh_size = generation_size (max_generation + 1);
32221 size_t total_soh_size = generation_sizes (generation_of (max_generation));
32223 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
32225 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
32226 generation_free_list_space (generation_of (max_generation)),
32227 generation_free_obj_space (generation_of (max_generation))));
32228 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
32230 generation_free_list_space (generation_of (max_generation + 1)),
32231 generation_free_obj_space (generation_of (max_generation + 1))));
32233 FIRE_EVENT(BGC2ndConEnd);
32234 concurrent_print_time_delta ("background sweep");
32236 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
32237 PREFIX_ASSUME(reset_seg != NULL);
32241 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
32242 heap_segment_background_allocated (reset_seg) = 0;
32243 reset_seg = heap_segment_next_rw (reset_seg);
32246 generation* loh_gen = generation_of (max_generation + 1);
32247 generation_allocation_segment (loh_gen) = heap_segment_rw (generation_start_segment (loh_gen));
32249 // We calculate dynamic data here because if we wait till we signal the lh event,
32250 // the allocation thread can change the fragmentation and we may read an intermediate
32251 // value (which can be greater than the generation size). Plus by that time it won't
32253 compute_new_dynamic_data (max_generation);
32255 enable_preemptive ();
32257 #ifdef MULTIPLE_HEAPS
32258 bgc_t_join.join(this, gc_join_set_state_free);
32259 if (bgc_t_join.joined())
32260 #endif //MULTIPLE_HEAPS
32262 // TODO: We are using this join just to set the state. Should
32263 // look into eliminating it - check to make sure things that use
32264 // this state can live with per heap state like should_check_bgc_mark.
32265 current_c_gc_state = c_gc_state_free;
32267 #ifdef MULTIPLE_HEAPS
32268 dprintf(2, ("Starting BGC threads after background sweep phase"));
32269 bgc_t_join.restart();
32270 #endif //MULTIPLE_HEAPS
32273 disable_preemptive (true);
32275 if (gc_lh_block_event.IsValid())
32277 gc_lh_block_event.Set();
32280 add_saved_spinlock_info (true, me_release, mt_bgc_loh_sweep);
32281 leave_spin_lock (&more_space_lock_loh);
32283 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
32284 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
32286 #endif //BACKGROUND_GC
32288 void gc_heap::sweep_large_objects ()
32290 //this min value is for the sake of the dynamic tuning.
32291 //so we know that we are not starting even if we have no
32293 generation* gen = large_object_generation;
32294 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
32296 PREFIX_ASSUME(start_seg != NULL);
32298 heap_segment* seg = start_seg;
32299 heap_segment* prev_seg = 0;
32300 uint8_t* o = generation_allocation_start (gen);
32301 int align_const = get_alignment_constant (FALSE);
32303 //Skip the generation gap object
32304 o = o + Align(size (o), align_const);
32306 uint8_t* plug_end = o;
32307 uint8_t* plug_start = o;
32309 generation_allocator (gen)->clear();
32310 generation_free_list_space (gen) = 0;
32311 generation_free_obj_space (gen) = 0;
32314 dprintf (3, ("sweeping large objects"));
32315 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
32317 (size_t)heap_segment_mem (seg),
32318 (size_t)heap_segment_allocated (seg),
32323 if (o >= heap_segment_allocated (seg))
32325 heap_segment* next_seg = heap_segment_next (seg);
32326 //delete the empty segment if not the only one
32327 if ((plug_end == heap_segment_mem (seg)) &&
32328 (seg != start_seg) && !heap_segment_read_only_p (seg))
32330 //prepare for deletion
32331 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
32333 heap_segment_next (prev_seg) = next_seg;
32334 heap_segment_next (seg) = freeable_large_heap_segment;
32335 freeable_large_heap_segment = seg;
32339 if (!heap_segment_read_only_p (seg))
32341 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
32342 heap_segment_allocated (seg) = plug_end;
32343 decommit_heap_segment_pages (seg, 0);
32352 o = heap_segment_mem (seg);
32354 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
32355 (size_t)heap_segment_mem (seg),
32356 (size_t)heap_segment_allocated (seg)));
32359 if (large_object_marked(o, TRUE))
32362 //everything between plug_end and plug_start is free
32363 thread_gap (plug_end, plug_start-plug_end, gen);
32368 o = o + AlignQword (size (o));
32369 if (o >= heap_segment_allocated (seg))
32373 m = large_object_marked (o, TRUE);
32376 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32380 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
32382 o = o + AlignQword (size (o));
32387 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32389 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32392 void gc_heap::relocate_in_large_objects ()
32394 relocate_args args;
32396 args.high = gc_high;
32397 args.last_plug = 0;
32399 generation* gen = large_object_generation;
32401 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32403 PREFIX_ASSUME(seg != NULL);
32405 uint8_t* o = generation_allocation_start (gen);
32409 if (o >= heap_segment_allocated (seg))
32411 seg = heap_segment_next_rw (seg);
32416 o = heap_segment_mem (seg);
32419 while (o < heap_segment_allocated (seg))
32421 check_class_object_demotion (o);
32422 if (contain_pointers (o))
32424 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
32425 go_through_object_nostart (method_table (o), o, size(o), pval,
32427 reloc_survivor_helper (pval);
32430 o = o + AlignQword (size (o));
32435 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
32438 uint8_t* low = gc_low;
32439 size_t end_card = 0;
32440 generation* oldest_gen = generation_of (max_generation+1);
32441 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
32443 PREFIX_ASSUME(seg != NULL);
32445 uint8_t* beg = generation_allocation_start (oldest_gen);
32446 uint8_t* end = heap_segment_allocated (seg);
32448 size_t cg_pointers_found = 0;
32450 size_t card_word_end = (card_of (align_on_card_word (end)) /
32455 size_t n_card_set = 0;
32456 uint8_t* next_boundary = (relocating ?
32457 generation_plan_allocation_start (generation_of (max_generation -1)) :
32460 uint8_t* nhigh = (relocating ?
32461 heap_segment_plan_allocated (ephemeral_heap_segment) :
32464 BOOL foundp = FALSE;
32465 uint8_t* start_address = 0;
32466 uint8_t* limit = 0;
32467 size_t card = card_of (beg);
32469 #ifdef BACKGROUND_GC
32470 BOOL consider_bgc_mark_p = FALSE;
32471 BOOL check_current_sweep_p = FALSE;
32472 BOOL check_saved_sweep_p = FALSE;
32473 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32474 #endif //BACKGROUND_GC
32476 size_t total_cards_cleared = 0;
32478 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
32479 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
32482 if ((o < end) && (card_of(o) > card))
32484 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
32485 if (cg_pointers_found == 0)
32487 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
32488 clear_cards (card, card_of((uint8_t*)o));
32489 total_cards_cleared += (card_of((uint8_t*)o) - card);
32491 n_eph +=cg_pointers_found;
32492 cg_pointers_found = 0;
32493 card = card_of ((uint8_t*)o);
32495 if ((o < end) &&(card >= end_card))
32497 foundp = find_card (card_table, card, card_word_end, end_card);
32500 n_card_set+= end_card - card;
32501 start_address = max (beg, card_address (card));
32503 limit = min (end, card_address (end_card));
32505 if ((!foundp) || (o >= end) || (card_address (card) >= end))
32507 if ((foundp) && (cg_pointers_found == 0))
32509 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
32510 (size_t)card_address(card+1)));
32511 clear_cards (card, card+1);
32512 total_cards_cleared += 1;
32514 n_eph +=cg_pointers_found;
32515 cg_pointers_found = 0;
32516 if ((seg = heap_segment_next_rw (seg)) != 0)
32518 #ifdef BACKGROUND_GC
32519 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32520 #endif //BACKGROUND_GC
32521 beg = heap_segment_mem (seg);
32522 end = compute_next_end (seg, low);
32523 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
32524 card = card_of (beg);
32535 assert (card_set_p (card));
32537 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
32538 card, (size_t)o, (size_t)limit));
32540 assert (Align (size (o)) >= Align (min_obj_size));
32541 size_t s = size (o);
32542 uint8_t* next_o = o + AlignQword (s);
32548 assert (Align (s) >= Align (min_obj_size));
32549 next_o = o + AlignQword (s);
32552 dprintf (4, ("|%Ix|", (size_t)o));
32553 if (next_o < start_address)
32558 #ifdef BACKGROUND_GC
32559 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
32563 #endif //BACKGROUND_GC
32565 #ifdef COLLECTIBLE_CLASS
32566 if (is_collectible(o))
32568 BOOL passed_end_card_p = FALSE;
32570 if (card_of (o) > card)
32572 passed_end_card_p = card_transition (o, end, card_word_end,
32576 foundp, start_address,
32577 limit, total_cards_cleared);
32580 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
32582 // card is valid and it covers the head of the object
32583 if (fn == &gc_heap::relocate_address)
32585 keep_card_live (o, n_gen, cg_pointers_found);
32589 uint8_t* class_obj = get_class_object (o);
32590 mark_through_cards_helper (&class_obj, n_gen,
32591 cg_pointers_found, fn,
32592 nhigh, next_boundary);
32596 if (passed_end_card_p)
32598 if (foundp && (card_address (card) < next_o))
32600 goto go_through_refs;
32610 #endif //COLLECTIBLE_CLASS
32612 if (contain_pointers (o))
32614 dprintf(3,("Going through %Ix", (size_t)o));
32616 go_through_object (method_table(o), o, s, poo,
32617 start_address, use_start, (o + s),
32619 if (card_of ((uint8_t*)poo) > card)
32621 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
32626 foundp, start_address,
32627 limit, total_cards_cleared);
32629 if (passed_end_card_p)
32631 if (foundp && (card_address (card) < next_o))
32635 if (ppstop <= (uint8_t**)start_address)
32637 else if (poo < (uint8_t**)start_address)
32638 {poo = (uint8_t**)start_address;}
32648 mark_through_cards_helper (poo, n_gen,
32649 cg_pointers_found, fn,
32650 nhigh, next_boundary);
32662 // compute the efficiency ratio of the card table
32665 generation_skip_ratio = min (((n_eph > 800) ?
32666 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
32667 generation_skip_ratio);
32669 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
32670 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
32674 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
32675 n_eph, n_gen, n_card_set, generation_skip_ratio));
32679 void gc_heap::descr_segment (heap_segment* seg )
32682 uint8_t* x = heap_segment_mem (seg);
32683 while (x < heap_segment_allocated (seg))
32685 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
32686 x = x + Align(size (x));
32689 UNREFERENCED_PARAMETER(seg);
32693 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
32695 #ifdef MULTIPLE_HEAPS
32696 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
32697 for (int i = 0; i < n_heaps; i++)
32699 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
32700 #else //MULTIPLE_HEAPS
32702 gc_heap* hp = NULL;
32704 // prefix complains about us dereferencing hp in wks build even though we only access static members
32705 // this way. not sure how to shut it up except for this ugly workaround:
32706 PREFIX_ASSUME(hp != NULL);
32707 #endif // _PREFAST_
32708 #endif //MULTIPLE_HEAPS
32710 int curr_gen_number0 = max_generation+1;
32711 while (curr_gen_number0 >= 0)
32713 generation* gen = hp->generation_of (curr_gen_number0);
32714 heap_segment* seg = generation_start_segment (gen);
32715 while (seg && (seg != hp->ephemeral_heap_segment))
32717 assert (curr_gen_number0 > 0);
32719 // report bounds from heap_segment_mem (seg) to
32720 // heap_segment_allocated (seg);
32721 // for generation # curr_gen_number0
32722 // for heap # heap_no
32724 fn(context, curr_gen_number0, heap_segment_mem (seg),
32725 heap_segment_allocated (seg),
32726 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
32728 seg = heap_segment_next (seg);
32732 assert (seg == hp->ephemeral_heap_segment);
32733 assert (curr_gen_number0 <= max_generation);
32735 if (curr_gen_number0 == max_generation)
32737 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32739 // report bounds from heap_segment_mem (seg) to
32740 // generation_allocation_start (generation_of (max_generation-1))
32741 // for heap # heap_number
32743 fn(context, curr_gen_number0, heap_segment_mem (seg),
32744 generation_allocation_start (hp->generation_of (max_generation-1)),
32745 generation_allocation_start (hp->generation_of (max_generation-1)) );
32748 else if (curr_gen_number0 != 0)
32750 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32751 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32752 // for heap # heap_number
32754 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32755 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32756 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32760 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32761 // to heap_segment_allocated (ephemeral_heap_segment);
32762 // for heap # heap_number
32764 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32765 heap_segment_allocated (hp->ephemeral_heap_segment),
32766 heap_segment_reserved (hp->ephemeral_heap_segment) );
32769 curr_gen_number0--;
32775 // Note that when logging is on it can take a long time to go through the free items.
32776 void gc_heap::print_free_list (int gen, heap_segment* seg)
32778 UNREFERENCED_PARAMETER(gen);
32779 UNREFERENCED_PARAMETER(seg);
32781 if (settings.concurrent == FALSE)
32783 uint8_t* seg_start = heap_segment_mem (seg);
32784 uint8_t* seg_end = heap_segment_allocated (seg);
32786 dprintf (3, ("Free list in seg %Ix:", seg_start));
32788 size_t total_free_item = 0;
32790 allocator* gen_allocator = generation_allocator (generation_of (gen));
32791 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32793 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32796 if (fo >= seg_start && fo < seg_end)
32800 size_t free_item_len = size(fo);
32802 dprintf (3, ("[%Ix, %Ix[:%Id",
32804 (size_t)(fo + free_item_len),
32808 fo = free_list_slot (fo);
32812 dprintf (3, ("total %Id free items", total_free_item));
32818 void gc_heap::descr_generations (BOOL begin_gc_p)
32820 UNREFERENCED_PARAMETER(begin_gc_p);
32822 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32825 #ifdef MULTIPLE_HEAPS
32827 #endif //MULTIPLE_HEAPS
32829 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32830 for (int n = max_generation; n >= 0; --n)
32832 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32834 generation_allocation_start(generation_of(n)),
32835 generation_allocation_limit(generation_of(n)),
32836 generation_allocation_pointer(generation_of(n)));
32838 heap_segment* seg = generation_start_segment(generation_of(n));
32841 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32842 heap_segment_mem(seg),
32843 heap_segment_allocated(seg),
32844 heap_segment_used(seg),
32845 heap_segment_committed(seg));
32846 seg = heap_segment_next(seg);
32850 #endif // STRESS_LOG
32853 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32854 (size_t) lowest_address, (size_t) highest_address));
32855 #ifdef BACKGROUND_GC
32856 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32857 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32858 #endif //BACKGROUND_GC
32860 if (heap_number == 0)
32862 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32865 int curr_gen_number = max_generation+1;
32866 while (curr_gen_number >= 0)
32868 size_t total_gen_size = generation_size (curr_gen_number);
32869 #ifdef SIMPLE_DPRINTF
32870 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32871 (begin_gc_p ? "BEG" : "END"),
32872 settings.condemned_generation,
32875 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32876 generation_free_list_space (generation_of (curr_gen_number)),
32877 generation_free_obj_space (generation_of (curr_gen_number)),
32879 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32881 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32882 (settings.heap_expansion ? "(EX)" : " "),
32883 (settings.promotion ? "Promotion" : "NoPromotion")));
32885 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32887 size (generation_allocation_start (generation_of (curr_gen_number))),
32889 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32890 #endif //SIMPLE_DPRINTF
32892 generation* gen = generation_of (curr_gen_number);
32893 heap_segment* seg = generation_start_segment (gen);
32894 while (seg && (seg != ephemeral_heap_segment))
32896 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32898 (size_t)heap_segment_mem (seg),
32899 (size_t)heap_segment_allocated (seg),
32900 (size_t)heap_segment_committed (seg),
32901 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32902 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32903 print_free_list (curr_gen_number, seg);
32904 seg = heap_segment_next (seg);
32906 if (seg && (seg != generation_start_segment (gen)))
32908 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32910 (size_t)heap_segment_mem (seg),
32911 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32912 print_free_list (curr_gen_number, seg);
32917 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32919 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32920 (size_t)(((curr_gen_number == 0)) ?
32921 (heap_segment_allocated
32922 (generation_start_segment
32923 (generation_of (curr_gen_number)))) :
32924 (generation_allocation_start
32925 (generation_of (curr_gen_number - 1))))
32927 print_free_list (curr_gen_number, seg);
32939 //-----------------------------------------------------------------------------
32941 // VM Specific support
32943 //-----------------------------------------------------------------------------
32948 unsigned int PromotedObjectCount = 0;
32949 unsigned int CreatedObjectCount = 0;
32950 unsigned int AllocDuration = 0;
32951 unsigned int AllocCount = 0;
32952 unsigned int AllocBigCount = 0;
32953 unsigned int AllocSmallCount = 0;
32954 unsigned int AllocStart = 0;
32957 //Static member variables.
32958 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32960 //CMCSafeLock* GCHeap::fGcLock;
32961 GCEvent *GCHeap::WaitForGCEvent = NULL;
32964 unsigned int GCHeap::GcDuration;
32966 unsigned GCHeap::GcCondemnedGeneration = 0;
32967 size_t GCHeap::totalSurvivedSize = 0;
32968 #ifdef FEATURE_PREMORTEM_FINALIZATION
32969 CFinalize* GCHeap::m_Finalize = 0;
32970 BOOL GCHeap::GcCollectClasses = FALSE;
32971 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32973 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32975 #ifdef BACKGROUND_GC
32976 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32977 #endif // BACKGROUND_GC
32978 #ifndef MULTIPLE_HEAPS
32979 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32980 int GCHeap::m_CurStressObj = 0;
32981 #endif // !MULTIPLE_HEAPS
32982 #endif // STRESS_HEAP
32983 #endif // FEATURE_REDHAWK
32985 #endif //FEATURE_PREMORTEM_FINALIZATION
32987 class NoGCRegionLockHolder
32990 NoGCRegionLockHolder()
32992 enter_spin_lock_noinstru(&g_no_gc_lock);
32995 ~NoGCRegionLockHolder()
32997 leave_spin_lock_noinstru(&g_no_gc_lock);
33001 // An explanation of locking for finalization:
33003 // Multiple threads allocate objects. During the allocation, they are serialized by
33004 // the AllocLock above. But they release that lock before they register the object
33005 // for finalization. That's because there is much contention for the alloc lock, but
33006 // finalization is presumed to be a rare case.
33008 // So registering an object for finalization must be protected by the FinalizeLock.
33010 // There is another logical queue that involves finalization. When objects registered
33011 // for finalization become unreachable, they are moved from the "registered" queue to
33012 // the "unreachable" queue. Note that this only happens inside a GC, so no other
33013 // threads can be manipulating either queue at that time. Once the GC is over and
33014 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
33015 // queue and call their finalizers. This dequeue operation is also protected with
33016 // the finalize lock.
33018 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
33019 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
33020 // when a GC is not in progress). The reason we share a lock with threads enqueuing
33021 // on the "registered" queue is that the "registered" and "unreachable" queues are
33024 // They are actually two regions of a longer list, which can only grow at one end.
33025 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
33026 // object at the boundary between the logical queues, out to the other end of the
33027 // unreachable queue -- where all growing takes place. Then you move the boundary
33028 // pointer so that the gap we created at the boundary is now on the "registered"
33029 // side rather than the "unreachable" side. Now the object can be placed into the
33030 // "registered" side at that point. This is much more efficient than doing moves
33031 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
33033 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
33034 // on the fact that the lock will only be taken for a brief period and that it will
33035 // never provoke or allow a GC while the lock is held. This is critical. If the
33036 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
33037 // allow a GC), then the Alloc client would have to GC protect a finalizable object
33038 // to protect against that eventuality. That is too slow!
33042 BOOL IsValidObject99(uint8_t *pObject)
33045 if (!((CObjectHeader*)pObject)->IsFree())
33046 ((CObjectHeader *) pObject)->Validate();
33047 #endif //VERIFY_HEAP
33051 #ifdef BACKGROUND_GC
33052 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
33054 uint8_t** range_beg,
33055 uint8_t** range_end)
33057 uint8_t* seg_start = heap_segment_mem (seg);
33058 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
33060 if ((seg_start < background_saved_highest_address) &&
33061 (seg_end > background_saved_lowest_address))
33063 *range_beg = max (seg_start, background_saved_lowest_address);
33064 *range_end = min (seg_end, background_saved_highest_address);
33073 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
33075 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33076 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33078 uint8_t* range_beg = 0;
33079 uint8_t* range_end = 0;
33081 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
33083 size_t markw = mark_word_of (range_beg);
33084 size_t markw_end = mark_word_of (range_end);
33085 while (markw < markw_end)
33087 if (mark_array [markw])
33089 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33090 markw, mark_array [markw], mark_word_address (markw)));
33095 uint8_t* p = mark_word_address (markw_end);
33096 while (p < range_end)
33098 assert (!(mark_array_marked (p)));
33103 #endif //VERIFY_HEAP && MARK_ARRAY
33106 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
33108 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33109 size_t start_mark_bit = mark_bit_of (obj) + 1;
33110 size_t end_mark_bit = mark_bit_of (obj + s);
33111 unsigned int startbit = mark_bit_bit (start_mark_bit);
33112 unsigned int endbit = mark_bit_bit (end_mark_bit);
33113 size_t startwrd = mark_bit_word (start_mark_bit);
33114 size_t endwrd = mark_bit_word (end_mark_bit);
33115 unsigned int result = 0;
33117 unsigned int firstwrd = ~(lowbits (~0, startbit));
33118 unsigned int lastwrd = ~(highbits (~0, endbit));
33120 if (startwrd == endwrd)
33122 unsigned int wrd = firstwrd & lastwrd;
33123 result = mark_array[startwrd] & wrd;
33131 // verify the first mark word is cleared.
33134 result = mark_array[startwrd] & firstwrd;
33142 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
33144 result = mark_array[wrdtmp];
33151 // set the last mark word.
33154 result = mark_array[endwrd] & lastwrd;
33160 #endif //VERIFY_HEAP && MARK_ARRAY
33163 void gc_heap::clear_all_mark_array()
33166 //size_t num_dwords_written = 0;
33167 //size_t begin_time = GetHighPrecisionTimeStamp();
33169 generation* gen = generation_of (max_generation);
33170 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33176 if (gen != large_object_generation)
33178 gen = generation_of (max_generation+1);
33179 seg = heap_segment_rw (generation_start_segment (gen));
33187 uint8_t* range_beg = 0;
33188 uint8_t* range_end = 0;
33190 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
33192 size_t markw = mark_word_of (range_beg);
33193 size_t markw_end = mark_word_of (range_end);
33194 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
33195 //num_dwords_written = markw_end - markw;
33197 size_t size_left = 0;
33199 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
33201 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
33203 size = (size_total & ~(sizeof(PTR_PTR) - 1));
33204 size_left = size_total - size;
33205 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
33212 memclr ((uint8_t*)&mark_array[markw], size);
33214 if (size_left != 0)
33216 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
33217 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
33219 *markw_to_clear = 0;
33225 seg = heap_segment_next_rw (seg);
33228 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
33230 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
33232 #endif //MARK_ARRAY
33235 #endif //BACKGROUND_GC
33237 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
33239 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33240 assert (card_table == g_gc_card_table);
33241 size_t markw = mark_word_of (heap_segment_mem (seg));
33242 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33244 while (markw < markw_end)
33246 if (mark_array [markw])
33248 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33249 markw, mark_array [markw], mark_word_address (markw)));
33254 #endif //VERIFY_HEAP && MARK_ARRAY
33257 void gc_heap::verify_mark_array_cleared ()
33259 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33260 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33262 generation* gen = generation_of (max_generation);
33263 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33269 if (gen != large_object_generation)
33271 gen = generation_of (max_generation+1);
33272 seg = heap_segment_rw (generation_start_segment (gen));
33280 bgc_verify_mark_array_cleared (seg);
33281 seg = heap_segment_next_rw (seg);
33284 #endif //VERIFY_HEAP && MARK_ARRAY
33287 void gc_heap::verify_seg_end_mark_array_cleared()
33289 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33290 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33292 generation* gen = generation_of (max_generation);
33293 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33299 if (gen != large_object_generation)
33301 gen = generation_of (max_generation+1);
33302 seg = heap_segment_rw (generation_start_segment (gen));
33310 // We already cleared all mark array bits for ephemeral generations
33311 // at the beginning of bgc sweep
33312 uint8_t* from = ((seg == ephemeral_heap_segment) ?
33313 generation_allocation_start (generation_of (max_generation - 1)) :
33314 heap_segment_allocated (seg));
33315 size_t markw = mark_word_of (align_on_mark_word (from));
33316 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33318 while (from < mark_word_address (markw))
33320 if (is_mark_bit_set (from))
33322 dprintf (3, ("mark bit for %Ix was not cleared", from));
33326 from += mark_bit_pitch;
33329 while (markw < markw_end)
33331 if (mark_array [markw])
33333 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33334 markw, mark_array [markw], mark_word_address (markw)));
33339 seg = heap_segment_next_rw (seg);
33342 #endif //VERIFY_HEAP && MARK_ARRAY
33345 // This function is called to make sure we don't mess up the segment list
33346 // in SOH. It's called by:
33347 // 1) begin and end of ephemeral GCs
33348 // 2) during bgc sweep when we switch segments.
33349 void gc_heap::verify_soh_segment_list()
33352 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33354 generation* gen = generation_of (max_generation);
33355 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33356 heap_segment* last_seg = 0;
33360 seg = heap_segment_next_rw (seg);
33362 if (last_seg != ephemeral_heap_segment)
33367 #endif //VERIFY_HEAP
33370 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
33371 // it can be called at the end of the final marking; and at any point during background
33373 // NOTE - to be able to call this function during background sweep, we need to temporarily
33374 // NOT clear the mark array bits as we go.
33375 void gc_heap::verify_partial ()
33377 #ifdef BACKGROUND_GC
33378 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
33379 //generation* gen = large_object_generation;
33380 generation* gen = generation_of (max_generation);
33381 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33382 int align_const = get_alignment_constant (gen != large_object_generation);
33388 // Different ways to fail.
33389 BOOL mark_missed_p = FALSE;
33390 BOOL bad_ref_p = FALSE;
33391 BOOL free_ref_p = FALSE;
33397 if (gen != large_object_generation)
33400 gen = large_object_generation;
33401 align_const = get_alignment_constant (gen != large_object_generation);
33402 seg = heap_segment_rw (generation_start_segment (gen));
33411 o = heap_segment_mem (seg);
33412 end = heap_segment_allocated (seg);
33413 //printf ("validating [%Ix-[%Ix\n", o, end);
33418 BOOL marked_p = background_object_marked (o, FALSE);
33422 go_through_object_cl (method_table (o), o, s, oo,
33426 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
33427 MethodTable *pMT = method_table (*oo);
33429 if (pMT == g_gc_pFreeObjectMethodTable)
33435 if (!pMT->SanityCheck())
33438 dprintf (3, ("Bad member of %Ix %Ix",
33439 (size_t)oo, (size_t)*oo));
33443 if (current_bgc_state == bgc_final_marking)
33445 if (marked_p && !background_object_marked (*oo, FALSE))
33447 mark_missed_p = TRUE;
33456 o = o + Align(s, align_const);
33458 seg = heap_segment_next_rw (seg);
33461 //printf ("didn't find any large object large enough...\n");
33462 //printf ("finished verifying loh\n");
33463 #endif //BACKGROUND_GC
33469 gc_heap::verify_free_lists ()
33471 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
33473 dprintf (3, ("Verifying free list for gen:%d", gen_num));
33474 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
33475 size_t sz = gen_alloc->first_bucket_size();
33476 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
33478 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
33480 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
33484 if (!((CObjectHeader*)free_list)->IsFree())
33486 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
33487 (size_t)free_list));
33490 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
33491 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
33493 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
33494 (size_t)free_list));
33497 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
33499 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
33500 (size_t)free_list));
33503 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
33505 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
33506 (size_t)free_list));
33511 free_list = free_list_slot (free_list);
33513 //verify the sanity of the tail
33514 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
33515 if (!((tail == 0) || (tail == prev)))
33517 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33522 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
33523 if ((head != 0) && (free_list_slot (head) != 0))
33525 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33536 gc_heap::verify_heap (BOOL begin_gc_p)
33538 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
33539 size_t last_valid_brick = 0;
33540 BOOL bCurrentBrickInvalid = FALSE;
33541 BOOL large_brick_p = TRUE;
33542 size_t curr_brick = 0;
33543 size_t prev_brick = (size_t)-1;
33544 int curr_gen_num = max_generation+1;
33545 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
33547 PREFIX_ASSUME(seg != NULL);
33549 uint8_t* curr_object = heap_segment_mem (seg);
33550 uint8_t* prev_object = 0;
33551 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
33552 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
33553 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
33554 int align_const = get_alignment_constant (FALSE);
33555 size_t total_objects_verified = 0;
33556 size_t total_objects_verified_deep = 0;
33558 #ifdef BACKGROUND_GC
33559 BOOL consider_bgc_mark_p = FALSE;
33560 BOOL check_current_sweep_p = FALSE;
33561 BOOL check_saved_sweep_p = FALSE;
33562 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33563 #endif //BACKGROUND_GC
33565 #ifdef MULTIPLE_HEAPS
33566 t_join* current_join = &gc_t_join;
33567 #ifdef BACKGROUND_GC
33568 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
33570 // We always call verify_heap on entry of GC on the SVR GC threads.
33571 current_join = &bgc_t_join;
33573 #endif //BACKGROUND_GC
33574 #endif //MULTIPLE_HEAPS
33576 UNREFERENCED_PARAMETER(begin_gc_p);
33577 #ifdef BACKGROUND_GC
33578 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
33579 (begin_gc_p ? "BEG" : "END"),
33580 VolatileLoad(&settings.gc_index),
33581 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33583 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
33584 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
33585 #endif //BACKGROUND_GC
33587 #ifndef MULTIPLE_HEAPS
33588 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
33589 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
33593 #endif //MULTIPLE_HEAPS
33595 #ifdef BACKGROUND_GC
33596 //don't touch the memory because the program is allocating from it.
33597 if (!settings.concurrent)
33598 #endif //BACKGROUND_GC
33600 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
33602 //uninit the unused portions of segments.
33603 generation* gen1 = large_object_generation;
33604 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
33605 PREFIX_ASSUME(seg1 != NULL);
33611 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
33612 if (heap_segment_used (seg1) > clear_start)
33614 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
33615 heap_segment_mem (seg1),
33617 heap_segment_used (seg1)));
33618 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
33619 (heap_segment_used (seg1) - clear_start));
33621 seg1 = heap_segment_next_rw (seg1);
33625 if (gen1 == large_object_generation)
33627 gen1 = generation_of (max_generation);
33628 seg1 = heap_segment_rw (generation_start_segment (gen1));
33629 PREFIX_ASSUME(seg1 != NULL);
33640 #ifdef MULTIPLE_HEAPS
33641 current_join->join(this, gc_join_verify_copy_table);
33642 if (current_join->joined())
33644 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
33645 for (int i = 0; i < n_heaps; i++)
33647 //copy the card and brick tables
33648 if (g_gc_card_table != g_heaps[i]->card_table)
33650 g_heaps[i]->copy_brick_card_table();
33654 current_join->restart();
33657 if (g_gc_card_table != card_table)
33658 copy_brick_card_table();
33659 #endif //MULTIPLE_HEAPS
33661 //verify that the generation structures makes sense
33663 generation* gen = generation_of (max_generation);
33665 assert (generation_allocation_start (gen) ==
33666 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
33667 int gen_num = max_generation-1;
33668 generation* prev_gen = gen;
33669 while (gen_num >= 0)
33671 gen = generation_of (gen_num);
33672 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
33673 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
33674 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
33676 if (generation_start_segment (prev_gen ) ==
33677 generation_start_segment (gen))
33679 assert (generation_allocation_start (prev_gen) <
33680 generation_allocation_start (gen));
33689 // Handle segment transitions
33690 if (curr_object >= heap_segment_allocated (seg))
33692 if (curr_object > heap_segment_allocated(seg))
33694 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
33695 (size_t)curr_object, (size_t)seg));
33698 seg = heap_segment_next_in_range (seg);
33701 #ifdef BACKGROUND_GC
33702 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33703 #endif //BACKGROUND_GC
33704 curr_object = heap_segment_mem(seg);
33710 if (curr_gen_num == (max_generation+1))
33713 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
33715 PREFIX_ASSUME(seg != NULL);
33717 #ifdef BACKGROUND_GC
33718 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33719 #endif //BACKGROUND_GC
33720 curr_object = heap_segment_mem (seg);
33722 large_brick_p = FALSE;
33723 align_const = get_alignment_constant (TRUE);
33726 break; // Done Verifying Heap -- no more segments
33730 // Are we at the end of the youngest_generation?
33731 if (seg == ephemeral_heap_segment)
33733 if (curr_object >= end_youngest)
33735 // prev_object length is too long if we hit this int3
33736 if (curr_object > end_youngest)
33738 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33739 (size_t)curr_object, (size_t)end_youngest));
33745 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33748 if (curr_gen_num > 0)
33750 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33755 //if (is_mark_set (curr_object))
33757 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33758 // FATAL_GC_ERROR();
33761 size_t s = size (curr_object);
33762 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33765 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33769 // If object is not in the youngest generation, then lets
33770 // verify that the brick table is correct....
33771 if (((seg != ephemeral_heap_segment) ||
33772 (brick_of(curr_object) < brick_of(begin_youngest))))
33774 curr_brick = brick_of(curr_object);
33776 // Brick Table Verification...
33778 // On brick transition
33779 // if brick is negative
33780 // verify that brick indirects to previous valid brick
33782 // set current brick invalid flag to be flipped if we
33783 // encounter an object at the correct place
33785 if (curr_brick != prev_brick)
33787 // If the last brick we were examining had positive
33788 // entry but we never found the matching object, then
33789 // we have a problem
33790 // If prev_brick was the last one of the segment
33791 // it's ok for it to be invalid because it is never looked at
33792 if (bCurrentBrickInvalid &&
33793 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33794 !heap_segment_read_only_p (seg))
33796 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33802 //large objects verify the table only if they are in
33804 if ((heap_segment_reserved (seg) <= highest_address) &&
33805 (heap_segment_mem (seg) >= lowest_address) &&
33806 brick_table [curr_brick] != 0)
33808 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33809 curr_brick, (size_t)curr_object));
33814 bCurrentBrickInvalid = FALSE;
33819 // If the current brick contains a negative value make sure
33820 // that the indirection terminates at the last valid brick
33821 if (brick_table [curr_brick] <= 0)
33823 if (brick_table [curr_brick] == 0)
33825 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33826 curr_brick, (size_t)curr_object));
33829 ptrdiff_t i = curr_brick;
33830 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33831 (brick_table[i] < 0))
33833 i = i + brick_table[i];
33835 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33837 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33838 i, brick_of (heap_segment_mem (seg)),
33842 // if (i != last_valid_brick)
33843 // FATAL_GC_ERROR();
33844 bCurrentBrickInvalid = FALSE;
33846 else if (!heap_segment_read_only_p (seg))
33848 bCurrentBrickInvalid = TRUE;
33853 if (bCurrentBrickInvalid)
33855 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33857 bCurrentBrickInvalid = FALSE;
33858 last_valid_brick = curr_brick;
33863 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33865 #ifdef FEATURE_LOH_COMPACTION
33866 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33868 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33870 #endif //FEATURE_LOH_COMPACTION
33872 total_objects_verified++;
33874 BOOL can_verify_deep = TRUE;
33875 #ifdef BACKGROUND_GC
33876 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33877 #endif //BACKGROUND_GC
33879 BOOL deep_verify_obj = can_verify_deep;
33880 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33881 deep_verify_obj = FALSE;
33883 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33885 if (can_verify_deep)
33887 if (curr_gen_num > 0)
33889 BOOL need_card_p = FALSE;
33890 if (contain_pointers_or_collectible (curr_object))
33892 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33893 size_t crd = card_of (curr_object);
33894 BOOL found_card_p = card_set_p (crd);
33896 #ifdef COLLECTIBLE_CLASS
33897 if (is_collectible(curr_object))
33899 uint8_t* class_obj = get_class_object (curr_object);
33900 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33904 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33905 card_of (curr_object), (size_t)curr_object, class_obj));
33911 #endif //COLLECTIBLE_CLASS
33913 if (contain_pointers(curr_object))
33915 go_through_object_nostart
33916 (method_table(curr_object), curr_object, s, oo,
33918 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33920 crd = card_of ((uint8_t*)oo);
33921 found_card_p = card_set_p (crd);
33922 need_card_p = FALSE;
33924 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33926 need_card_p = TRUE;
33929 if (need_card_p && !found_card_p)
33932 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33933 card_of (curr_object), (size_t)curr_object,
33934 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33940 if (need_card_p && !found_card_p)
33942 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33943 card_of (curr_object), (size_t)curr_object,
33944 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33949 total_objects_verified_deep++;
33953 prev_object = curr_object;
33954 prev_brick = curr_brick;
33955 curr_object = curr_object + Align(s, align_const);
33956 if (curr_object < prev_object)
33958 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33963 #ifdef BACKGROUND_GC
33964 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33965 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33966 (begin_gc_p ? "BEG" : "END"),
33967 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33968 total_objects_verified, total_objects_verified_deep));
33969 if (current_c_gc_state != c_gc_state_planning)
33971 assert (total_objects_verified == total_objects_verified_deep);
33973 #endif //BACKGROUND_GC
33975 verify_free_lists();
33977 #ifdef FEATURE_PREMORTEM_FINALIZATION
33978 finalize_queue->CheckFinalizerObjects();
33979 #endif // FEATURE_PREMORTEM_FINALIZATION
33982 // to be consistent with handle table APIs pass a ScanContext*
33983 // to provide the heap number. the SC isn't complete though so
33984 // limit its scope to handle table verification.
33986 sc.thread_number = heap_number;
33987 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33990 #ifdef MULTIPLE_HEAPS
33991 current_join->join(this, gc_join_verify_objects_done);
33992 if (current_join->joined())
33993 #endif //MULTIPLE_HEAPS
33995 GCToEEInterface::VerifySyncTableEntry();
33996 #ifdef MULTIPLE_HEAPS
33997 current_join->restart();
33998 #endif //MULTIPLE_HEAPS
34001 #ifdef BACKGROUND_GC
34002 if (!settings.concurrent)
34004 if (current_c_gc_state == c_gc_state_planning)
34006 // temporarily commenting this out 'cause an FGC
34007 // could be triggered before we sweep ephemeral.
34008 //verify_seg_end_mark_array_cleared();
34012 if (settings.concurrent)
34014 verify_mark_array_cleared();
34016 dprintf (2,("GC%d(%s): Verifying heap - end",
34017 VolatileLoad(&settings.gc_index),
34018 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
34020 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
34021 #endif //BACKGROUND_GC
34024 #endif //VERIFY_HEAP
34027 void GCHeap::ValidateObjectMember (Object* obj)
34030 size_t s = size (obj);
34031 uint8_t* o = (uint8_t*)obj;
34033 go_through_object_cl (method_table (obj), o, s, oo,
34035 uint8_t* child_o = *oo;
34038 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
34039 MethodTable *pMT = method_table (child_o);
34041 if (!pMT->SanityCheck()) {
34042 dprintf (3, ("Bad member of %Ix %Ix",
34043 (size_t)oo, (size_t)child_o));
34048 #endif // VERIFY_HEAP
34051 void DestructObject (CObjectHeader* hdr)
34053 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
34054 hdr->~CObjectHeader();
34057 HRESULT GCHeap::Shutdown ()
34061 GCScan::GcRuntimeStructuresValid (FALSE);
34063 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
34064 // threads except the one performing the shutdown.
34065 // ASSERT( !GcInProgress );
34067 // Guard against any more GC occurring and against any threads blocking
34068 // for GC to complete when the GC heap is gone. This fixes a race condition
34069 // where a thread in GC is destroyed as part of process destruction and
34070 // the remaining threads block for GC complete.
34073 //EnterAllocLock();
34075 //EnterFinalizeLock();
34078 // during shutdown lot of threads are suspended
34079 // on this even, we don't want to wake them up just yet
34080 //CloseHandle (WaitForGCEvent);
34082 //find out if the global card table hasn't been used yet
34083 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
34084 if (card_table_refcount (ct) == 0)
34086 destroy_card_table (ct);
34087 g_gc_card_table = nullptr;
34089 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
34090 g_gc_card_bundle_table = nullptr;
34092 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34093 SoftwareWriteWatch::StaticClose();
34094 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34097 //destroy all segments on the standby list
34098 while(gc_heap::segment_standby_list != 0)
34100 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
34101 #ifdef MULTIPLE_HEAPS
34102 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34103 #else //MULTIPLE_HEAPS
34104 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34105 #endif //MULTIPLE_HEAPS
34106 gc_heap::segment_standby_list = next_seg;
34110 #ifdef MULTIPLE_HEAPS
34112 for (int i = 0; i < gc_heap::n_heaps; i ++)
34114 delete gc_heap::g_heaps[i]->vm_heap;
34115 //destroy pure GC stuff
34116 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
34119 gc_heap::destroy_gc_heap (pGenGCHeap);
34121 #endif //MULTIPLE_HEAPS
34122 gc_heap::shutdown_gc();
34127 // Wait until a garbage collection is complete
34128 // returns NOERROR if wait was OK, other error code if failure.
34129 // WARNING: This will not undo the must complete state. If you are
34130 // in a must complete when you call this, you'd better know what you're
34133 #ifdef FEATURE_PREMORTEM_FINALIZATION
34135 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
34137 *pCFinalize = new (nothrow) CFinalize();
34138 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
34139 return E_OUTOFMEMORY;
34143 #endif // FEATURE_PREMORTEM_FINALIZATION
34145 // init the instance heap
34146 HRESULT GCHeap::Init(size_t hn)
34148 HRESULT hres = S_OK;
34150 #ifdef MULTIPLE_HEAPS
34151 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
34152 hres = E_OUTOFMEMORY;
34154 UNREFERENCED_PARAMETER(hn);
34155 if (!gc_heap::make_gc_heap())
34156 hres = E_OUTOFMEMORY;
34157 #endif //MULTIPLE_HEAPS
34163 //System wide initialization
34164 HRESULT GCHeap::Initialize()
34168 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
34169 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
34170 assert(g_num_processors != 0);
34172 //Initialize the static members.
34175 CreatedObjectCount = 0;
34178 bool is_restricted;
34179 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted);
34182 gc_heap::heap_hard_limit = (size_t)GCConfig::GetGCHeapHardLimit();
34184 if (!(gc_heap::heap_hard_limit))
34186 uint32_t percent_of_mem = (uint32_t)GCConfig::GetGCHeapHardLimitPercent();
34187 if ((percent_of_mem > 0) && (percent_of_mem < 100))
34189 gc_heap::heap_hard_limit = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem / (uint64_t)100);
34193 // If the hard limit is specified, the user is saying even if the process is already
34194 // running in a container, use this limit for the GC heap.
34195 if (!(gc_heap::heap_hard_limit))
34199 uint64_t physical_mem_for_gc = gc_heap::total_physical_mem * (uint64_t)75 / (uint64_t)100;
34200 //printf ("returned physical mem %I64d, setting it to max (75%%: %I64d, 20mb)\n",
34201 // gc_heap::total_physical_mem, physical_mem_for_gc);
34202 gc_heap::heap_hard_limit = (size_t)max ((20 * 1024 * 1024), physical_mem_for_gc);
34206 //printf ("heap_hard_limit is %Id, total physical mem: %Id, %s restricted\n",
34207 // gc_heap::heap_hard_limit, gc_heap::total_physical_mem, (is_restricted ? "is" : "is not"));
34211 uint32_t nhp_from_config = 0;
34213 #ifdef MULTIPLE_HEAPS
34214 nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
34216 // GetCurrentProcessCpuCount only returns up to 64 procs.
34217 uint32_t nhp_from_process = GCToOSInterface::CanEnableGCCPUGroups() ?
34218 GCToOSInterface::GetTotalProcessorCount():
34219 GCToOSInterface::GetCurrentProcessCpuCount();
34221 if (nhp_from_config)
34223 // Even when the user specifies a heap count, it should not be more
34224 // than the number of procs this process can use.
34225 nhp_from_config = min (nhp_from_config, nhp_from_process);
34228 nhp = ((nhp_from_config == 0) ? nhp_from_process : nhp_from_config);
34230 nhp = min (nhp, MAX_SUPPORTED_CPUS);
34231 #ifndef FEATURE_REDHAWK
34232 gc_heap::gc_thread_no_affinitize_p = (gc_heap::heap_hard_limit ? false : (GCConfig::GetNoAffinitize() != 0));
34234 size_t gc_thread_affinity_mask = static_cast<size_t>(GCConfig::GetGCHeapAffinitizeMask());
34236 if (gc_heap::heap_hard_limit)
34238 gc_heap::gc_thread_no_affinitize_p = (gc_thread_affinity_mask == 0);
34241 if (!(gc_heap::gc_thread_no_affinitize_p))
34243 if (!(GCToOSInterface::CanEnableGCCPUGroups()))
34245 uintptr_t pmask, smask;
34246 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
34251 // GetCurrentProcessAffinityMask can return pmask=0 and smask=0 on
34252 // systems with more than 1 NUMA node. The pmask decides the
34253 // number of GC heaps to be used and the processors they are
34254 // affinitized with. So pmask is now set to reflect that 64
34255 // processors are available to begin with. The actual processors in
34256 // the system may be lower and are taken into account before
34257 // finalizing the number of heaps.
34260 pmask = SIZE_T_MAX;
34262 #endif // FEATURE_PAL
34264 if (gc_thread_affinity_mask)
34266 pmask &= gc_thread_affinity_mask;
34269 process_mask = pmask;
34271 unsigned int set_bits_in_pmask = 0;
34275 set_bits_in_pmask++;
34279 nhp = min (nhp, set_bits_in_pmask);
34282 // Limit the GC heaps to the number of processors available in the system.
34283 nhp = min (nhp, GCToOSInterface::GetTotalProcessorCount());
34284 #endif // FEATURE_PAL
34288 gc_heap::gc_thread_no_affinitize_p = true;
34292 #endif //!FEATURE_REDHAWK
34293 #endif //MULTIPLE_HEAPS
34295 size_t seg_size = 0;
34296 size_t large_seg_size = 0;
34298 if (gc_heap::heap_hard_limit)
34300 seg_size = gc_heap::get_segment_size_hard_limit (&nhp, (nhp_from_config == 0));
34301 gc_heap::soh_segment_size = seg_size;
34302 large_seg_size = seg_size * 2;
34306 seg_size = get_valid_segment_size();
34307 gc_heap::soh_segment_size = seg_size;
34308 large_seg_size = get_valid_segment_size (TRUE);
34311 dprintf (1, ("%d heaps, soh seg size: %Id mb, loh: %Id mb\n",
34313 (seg_size / (size_t)1024 / 1024),
34314 (large_seg_size / 1024 / 1024)));
34316 gc_heap::min_loh_segment_size = large_seg_size;
34317 gc_heap::min_segment_size = min (seg_size, large_seg_size);
34318 #ifdef SEG_MAPPING_TABLE
34319 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
34320 #endif //SEG_MAPPING_TABLE
34322 #ifdef MULTIPLE_HEAPS
34323 gc_heap::n_heaps = nhp;
34324 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
34326 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
34327 #endif //MULTIPLE_HEAPS
34332 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
34333 #ifndef MULTIPLE_HEAPS
34334 gc_heap::mem_one_percent /= g_num_processors;
34335 #endif //!MULTIPLE_HEAPS
34337 uint32_t highmem_th_from_config = (uint32_t)GCConfig::GetGCHighMemPercent();
34338 if (highmem_th_from_config)
34340 gc_heap::high_memory_load_th = min (99, highmem_th_from_config);
34341 gc_heap::v_high_memory_load_th = min (99, (highmem_th_from_config + 7));
34345 // We should only use this if we are in the "many process" mode which really is only applicable
34346 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
34347 // For now I am using an estimate to calculate these numbers but this should really be obtained
34348 // programmatically going forward.
34349 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
34350 // I am assuming 3 in part due to the "very high memory load" is 97%.
34351 int available_mem_th = 10;
34352 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
34354 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(GCToOSInterface::GetTotalProcessorCount()));
34355 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
34358 gc_heap::high_memory_load_th = 100 - available_mem_th;
34359 gc_heap::v_high_memory_load_th = 97;
34362 gc_heap::m_high_memory_load_th = min ((gc_heap::high_memory_load_th + 5), gc_heap::v_high_memory_load_th);
34364 gc_heap::pm_stress_on = (GCConfig::GetGCProvModeStress() != 0);
34367 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
34370 WaitForGCEvent = new (nothrow) GCEvent;
34372 if (!WaitForGCEvent)
34374 return E_OUTOFMEMORY;
34377 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
34382 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34383 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
34384 if (GCStress<cfg_any>::IsEnabled()) {
34385 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
34387 m_StressObjs[i] = CreateGlobalHandle(0);
34389 m_CurStressObj = 0;
34391 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
34392 #endif // FEATURE_REDHAWK
34394 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
34396 #ifdef MULTIPLE_HEAPS
34398 for (unsigned i = 0; i < nhp; i++)
34400 GCHeap* Hp = new (nothrow) GCHeap();
34402 return E_OUTOFMEMORY;
34404 if ((hr = Hp->Init (i))!= S_OK)
34409 // initialize numa node to heap map
34410 heap_select::init_numa_node_to_heap_map(nhp);
34413 #endif //MULTIPLE_HEAPS
34417 GCScan::GcRuntimeStructuresValid (TRUE);
34419 GCToEEInterface::DiagUpdateGenerationBounds();
34426 // GC callback functions
34427 bool GCHeap::IsPromoted(Object* object)
34430 ((CObjectHeader*)object)->Validate();
34433 uint8_t* o = (uint8_t*)object;
34435 if (gc_heap::settings.condemned_generation == max_generation)
34437 #ifdef MULTIPLE_HEAPS
34438 gc_heap* hp = gc_heap::g_heaps[0];
34440 gc_heap* hp = pGenGCHeap;
34441 #endif //MULTIPLE_HEAPS
34443 #ifdef BACKGROUND_GC
34444 if (gc_heap::settings.concurrent)
34446 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
34447 hp->background_marked (o));
34451 #endif //BACKGROUND_GC
34453 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
34454 || hp->is_mark_set (o));
34459 gc_heap* hp = gc_heap::heap_of (o);
34460 return (!((o < hp->gc_high) && (o >= hp->gc_low))
34461 || hp->is_mark_set (o));
34465 size_t GCHeap::GetPromotedBytes(int heap_index)
34467 #ifdef BACKGROUND_GC
34468 if (gc_heap::settings.concurrent)
34470 return gc_heap::bpromoted_bytes (heap_index);
34473 #endif //BACKGROUND_GC
34475 return gc_heap::promoted_bytes (heap_index);
34479 void GCHeap::SetYieldProcessorScalingFactor (float scalingFactor)
34481 assert (yp_spin_count_unit != 0);
34482 int saved_yp_spin_count_unit = yp_spin_count_unit;
34483 yp_spin_count_unit = (int)((float)yp_spin_count_unit * scalingFactor / (float)9);
34485 // It's very suspicious if it becomes 0
34486 if (yp_spin_count_unit == 0)
34488 yp_spin_count_unit = saved_yp_spin_count_unit;
34492 unsigned int GCHeap::WhichGeneration (Object* object)
34494 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
34495 unsigned int g = hp->object_gennum ((uint8_t*)object);
34496 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
34500 bool GCHeap::IsEphemeral (Object* object)
34502 uint8_t* o = (uint8_t*)object;
34503 gc_heap* hp = gc_heap::heap_of (o);
34504 return !!hp->ephemeral_pointer_p (o);
34507 // Return NULL if can't find next object. When EE is not suspended,
34508 // the result is not accurate: if the input arg is in gen0, the function could
34509 // return zeroed out memory as next object
34510 Object * GCHeap::NextObj (Object * object)
34513 uint8_t* o = (uint8_t*)object;
34515 #ifndef FEATURE_BASICFREEZE
34516 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
34520 #endif //!FEATURE_BASICFREEZE
34522 heap_segment * hs = gc_heap::find_segment (o, FALSE);
34528 BOOL large_object_p = heap_segment_loh_p (hs);
34529 if (large_object_p)
34530 return NULL; //could be racing with another core allocating.
34531 #ifdef MULTIPLE_HEAPS
34532 gc_heap* hp = heap_segment_heap (hs);
34533 #else //MULTIPLE_HEAPS
34535 #endif //MULTIPLE_HEAPS
34536 unsigned int g = hp->object_gennum ((uint8_t*)object);
34537 if ((g == 0) && hp->settings.demotion)
34538 return NULL;//could be racing with another core allocating.
34539 int align_const = get_alignment_constant (!large_object_p);
34540 uint8_t* nextobj = o + Align (size (o), align_const);
34541 if (nextobj <= o) // either overflow or 0 sized object.
34546 if ((nextobj < heap_segment_mem(hs)) ||
34547 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
34548 (nextobj >= hp->alloc_allocated))
34553 return (Object *)nextobj;
34556 #endif // VERIFY_HEAP
34559 // returns TRUE if the pointer is in one of the GC heaps.
34560 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
34562 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
34563 // no longer calls GCEvent::Wait which eventually takes a lock.
34565 uint8_t* object = (uint8_t*) vpObject;
34566 #ifndef FEATURE_BASICFREEZE
34567 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
34569 #endif //!FEATURE_BASICFREEZE
34571 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
34575 #ifdef STRESS_PINNING
34576 static n_promote = 0;
34577 #endif //STRESS_PINNING
34578 // promote an object
34579 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
34581 THREAD_NUMBER_FROM_CONTEXT;
34582 #ifndef MULTIPLE_HEAPS
34583 const int thread = 0;
34584 #endif //!MULTIPLE_HEAPS
34586 uint8_t* o = (uint8_t*)*ppObject;
34591 #ifdef DEBUG_DestroyedHandleValue
34592 // we can race with destroy handle during concurrent scan
34593 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
34595 #endif //DEBUG_DestroyedHandleValue
34599 gc_heap* hp = gc_heap::heap_of (o);
34601 dprintf (3, ("Promote %Ix", (size_t)o));
34603 #ifdef INTERIOR_POINTERS
34604 if (flags & GC_CALL_INTERIOR)
34606 if ((o < hp->gc_low) || (o >= hp->gc_high))
34610 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
34616 #endif //INTERIOR_POINTERS
34618 #ifdef FEATURE_CONSERVATIVE_GC
34619 // For conservative GC, a value on stack may point to middle of a free object.
34620 // In this case, we don't need to promote the pointer.
34621 if (GCConfig::GetConservativeGC()
34622 && ((CObjectHeader*)o)->IsFree())
34629 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
34631 UNREFERENCED_PARAMETER(sc);
34634 if (flags & GC_CALL_PINNED)
34635 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34637 #ifdef STRESS_PINNING
34638 if ((++n_promote % 20) == 1)
34639 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34640 #endif //STRESS_PINNING
34642 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34643 size_t promoted_size_begin = hp->promoted_bytes (thread);
34644 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34646 if ((o >= hp->gc_low) && (o < hp->gc_high))
34648 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
34651 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34652 size_t promoted_size_end = hp->promoted_bytes (thread);
34653 if (g_fEnableAppDomainMonitoring)
34655 if (sc->pCurrentDomain)
34657 GCToEEInterface::RecordSurvivedBytesForHeap((promoted_size_end - promoted_size_begin), thread, sc->pCurrentDomain);
34660 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34662 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
34665 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
34668 UNREFERENCED_PARAMETER(sc);
34670 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
34672 THREAD_NUMBER_FROM_CONTEXT;
34674 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
34675 dprintf (3, ("R: %Ix", (size_t)ppObject));
34680 gc_heap* hp = gc_heap::heap_of (object);
34683 if (!(flags & GC_CALL_INTERIOR))
34685 // We cannot validate this object if it's in the condemned gen because it could
34686 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
34687 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34689 ((CObjectHeader*)object)->Validate(FALSE);
34694 dprintf (3, ("Relocate %Ix\n", (size_t)object));
34698 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
34700 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34705 if (gc_heap::loh_object_p (object))
34707 pheader = hp->find_object (object, 0);
34713 ptrdiff_t ref_offset = object - pheader;
34714 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34715 *ppObject = (Object*)(pheader + ref_offset);
34722 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34723 *ppObject = (Object*)pheader;
34726 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
34729 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
34731 // For now we simply look at the size of the object to determine if it in the
34732 // fixed heap or not. If the bit indicating this gets set at some point
34733 // we should key off that instead.
34734 return size( pObj ) >= loh_size_threshold;
34737 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34740 void StressHeapDummy ();
34742 static int32_t GCStressStartCount = -1;
34743 static int32_t GCStressCurCount = 0;
34744 static int32_t GCStressStartAtJit = -1;
34746 // the maximum number of foreground GCs we'll induce during one BGC
34747 // (this number does not include "naturally" occuring GCs).
34748 static int32_t GCStressMaxFGCsPerBGC = -1;
34750 // CLRRandom implementation can produce FPU exceptions if
34751 // the test/application run by CLR is enabling any FPU exceptions.
34752 // We want to avoid any unexpected exception coming from stress
34753 // infrastructure, so CLRRandom is not an option.
34754 // The code below is a replicate of CRT rand() implementation.
34755 // Using CRT rand() is not an option because we will interfere with the user application
34756 // that may also use it.
34757 int StressRNG(int iMaxValue)
34759 static BOOL bisRandInit = FALSE;
34760 static int lHoldrand = 1L;
34764 lHoldrand = (int)time(NULL);
34765 bisRandInit = TRUE;
34767 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
34768 return randValue % iMaxValue;
34770 #endif // STRESS_HEAP
34771 #endif // !FEATURE_REDHAWK
34773 // free up object so that things will move and then do a GC
34774 //return TRUE if GC actually happens, otherwise FALSE
34775 bool GCHeap::StressHeap(gc_alloc_context * context)
34777 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34778 alloc_context* acontext = static_cast<alloc_context*>(context);
34779 assert(context != nullptr);
34781 // if GC stress was dynamically disabled during this run we return FALSE
34782 if (!GCStressPolicy::IsEnabled())
34786 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
34792 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
34794 || g_pConfig->FastGCStressLevel() > 1
34797 if (!Thread::UniqueStack(&acontext)) {
34802 #ifdef BACKGROUND_GC
34803 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
34804 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
34808 #endif //BACKGROUND_GC
34810 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
34812 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
34813 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
34816 if (GCStressMaxFGCsPerBGC == -1)
34818 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
34819 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
34820 GCStressMaxFGCsPerBGC = 6;
34824 if (g_JitCount < GCStressStartAtJit)
34828 // Allow programmer to skip the first N Stress GCs so that you can
34829 // get to the interesting ones faster.
34830 Interlocked::Increment(&GCStressCurCount);
34831 if (GCStressCurCount < GCStressStartCount)
34834 // throttle the number of stress-induced GCs by a factor given by GCStressStep
34835 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
34840 #ifdef BACKGROUND_GC
34841 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
34843 // allow a maximum number of stress induced FGCs during one BGC
34844 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
34846 ++gc_stress_fgcs_in_bgc;
34848 #endif // BACKGROUND_GC
34850 if (g_pStringClass == 0)
34852 // If the String class has not been loaded, dont do any stressing. This should
34853 // be kept to a minimum to get as complete coverage as possible.
34854 _ASSERTE(g_fEEInit);
34858 #ifndef MULTIPLE_HEAPS
34859 static int32_t OneAtATime = -1;
34861 // Only bother with this if the stress level is big enough and if nobody else is
34862 // doing it right now. Note that some callers are inside the AllocLock and are
34863 // guaranteed synchronized. But others are using AllocationContexts and have no
34864 // particular synchronization.
34866 // For this latter case, we want a very high-speed way of limiting this to one
34867 // at a time. A secondary advantage is that we release part of our StressObjs
34868 // buffer sparingly but just as effectively.
34870 if (Interlocked::Increment(&OneAtATime) == 0 &&
34871 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34875 // If the current string is used up
34876 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34878 // Populate handles with strings
34879 int i = m_CurStressObj;
34880 while(HndFetchHandle(m_StressObjs[i]) == 0)
34882 _ASSERTE(m_StressObjs[i] != 0);
34883 unsigned strLen = ((unsigned)loh_size_threshold - 32) / sizeof(WCHAR);
34884 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34886 // update the cached type handle before allocating
34887 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34888 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34891 str->SetMethodTable (g_pStringClass);
34892 str->SetStringLength (strLen);
34893 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34895 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34896 if (i == m_CurStressObj) break;
34899 // advance the current handle to the next string
34900 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34903 // Get the current string
34904 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34907 // Chop off the end of the string and form a new object out of it.
34908 // This will 'free' an object at the begining of the heap, which will
34909 // force data movement. Note that we can only do this so many times.
34910 // before we have to move on to the next string.
34911 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34912 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34914 unsigned sizeToNextObj = (unsigned)Align(size(str));
34915 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34916 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34917 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34921 // Let the string itself become garbage.
34922 // will be realloced next time around
34923 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34927 Interlocked::Decrement(&OneAtATime);
34928 #endif // !MULTIPLE_HEAPS
34929 if (IsConcurrentGCEnabled())
34931 int rgen = StressRNG(10);
34933 // gen0:gen1:gen2 distribution: 40:40:20
34936 else if (rgen >= 4)
34941 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34945 GarbageCollect(max_generation, FALSE, collection_gcstress);
34950 UNREFERENCED_PARAMETER(context);
34952 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34956 #ifdef FEATURE_PREMORTEM_FINALIZATION
34957 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34958 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34959 #else // FEATURE_PREMORTEM_FINALIZATION
34960 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34961 #endif // FEATURE_PREMORTEM_FINALIZATION
34963 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34964 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34966 STRESS_LOG_OOM_STACK(_size); \
34972 // Small Object Allocator
34975 // Allocate small object with an alignment requirement of 8-bytes.
34977 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34979 #ifdef FEATURE_64BIT_ALIGNMENT
34985 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34987 #ifdef MULTIPLE_HEAPS
34988 if (acontext->get_alloc_heap() == 0)
34990 AssignHeap (acontext);
34991 assert (acontext->get_alloc_heap());
34994 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34996 gc_heap* hp = pGenGCHeap;
34997 #endif //MULTIPLE_HEAPS
34999 return AllocAlign8Common(hp, acontext, size, flags);
35001 UNREFERENCED_PARAMETER(ctx);
35002 UNREFERENCED_PARAMETER(size);
35003 UNREFERENCED_PARAMETER(flags);
35004 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
35006 #endif //FEATURE_64BIT_ALIGNMENT
35009 // Common code used by both variants of AllocAlign8 above.
35011 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
35013 #ifdef FEATURE_64BIT_ALIGNMENT
35019 gc_heap* hp = (gc_heap*)_hp;
35023 Object* newAlloc = NULL;
35026 #ifdef COUNT_CYCLES
35027 AllocStart = GetCycleCount32();
35029 #elif defined(ENABLE_INSTRUMENTATION)
35030 unsigned AllocStart = GetInstLogTime();
35032 #endif //COUNT_CYCLES
35035 if (size < loh_size_threshold)
35041 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
35042 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
35043 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
35044 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
35046 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
35047 // lock at this point).
35048 uint8_t* result = acontext->alloc_ptr;
35050 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
35052 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
35054 // Yes, we can just go ahead and make the allocation.
35055 newAlloc = (Object*) hp->allocate (size, acontext);
35056 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35060 // No, either the next available address is not aligned in the way we require it or there's
35061 // not enough space to allocate an object of the required size. In both cases we allocate a
35062 // padding object (marked as a free object). This object's size is such that it will reverse
35063 // the alignment of the next header (asserted below).
35065 // We allocate both together then decide based on the result whether we'll format the space as
35066 // free object + real object or real object + free object.
35067 ASSERT((Align(min_obj_size) & 7) == 4);
35068 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
35071 if (((size_t)freeobj & 7) == desiredAlignment)
35073 // New allocation has desired alignment, return this one and place the free object at the
35074 // end of the allocated space.
35075 newAlloc = (Object*)freeobj;
35076 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
35080 // New allocation is still mis-aligned, format the initial space as a free object and the
35081 // rest of the space should be correctly aligned for the real object.
35082 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
35083 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35085 freeobj->SetFree(min_obj_size);
35091 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
35092 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
35093 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
35094 // these can never get large enough to be allocated on the LOH.
35095 ASSERT(65536 < loh_size_threshold);
35096 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
35098 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35100 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
35101 ASSERT(((size_t)newAlloc & 7) == 0);
35104 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35107 #ifdef COUNT_CYCLES
35108 finish = GetCycleCount32();
35109 #elif defined(ENABLE_INSTRUMENTATION)
35110 finish = GetInstLogTime();
35111 #endif //COUNT_CYCLES
35112 AllocDuration += finish - AllocStart;
35117 UNREFERENCED_PARAMETER(_hp);
35118 UNREFERENCED_PARAMETER(acontext);
35119 UNREFERENCED_PARAMETER(size);
35120 UNREFERENCED_PARAMETER(flags);
35121 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
35123 #endif // FEATURE_64BIT_ALIGNMENT
35127 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
35136 Object* newAlloc = NULL;
35139 #ifdef COUNT_CYCLES
35140 AllocStart = GetCycleCount32();
35142 #elif defined(ENABLE_INSTRUMENTATION)
35143 unsigned AllocStart = GetInstLogTime();
35145 #endif //COUNT_CYCLES
35148 #ifdef MULTIPLE_HEAPS
35149 //take the first heap....
35150 gc_heap* hp = gc_heap::g_heaps[0];
35152 gc_heap* hp = pGenGCHeap;
35154 // prefix complains about us dereferencing hp in wks build even though we only access static members
35155 // this way. not sure how to shut it up except for this ugly workaround:
35156 PREFIX_ASSUME(hp != NULL);
35158 #endif //MULTIPLE_HEAPS
35160 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35162 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35163 #ifdef FEATURE_STRUCTALIGN
35164 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35165 #endif // FEATURE_STRUCTALIGN
35166 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35169 #ifdef COUNT_CYCLES
35170 finish = GetCycleCount32();
35171 #elif defined(ENABLE_INSTRUMENTATION)
35172 finish = GetInstLogTime();
35173 #endif //COUNT_CYCLES
35174 AllocDuration += finish - AllocStart;
35181 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
35190 Object* newAlloc = NULL;
35191 alloc_context* acontext = static_cast<alloc_context*>(context);
35194 #ifdef COUNT_CYCLES
35195 AllocStart = GetCycleCount32();
35197 #elif defined(ENABLE_INSTRUMENTATION)
35198 unsigned AllocStart = GetInstLogTime();
35200 #endif //COUNT_CYCLES
35203 #ifdef MULTIPLE_HEAPS
35204 if (acontext->get_alloc_heap() == 0)
35206 AssignHeap (acontext);
35207 assert (acontext->get_alloc_heap());
35209 #endif //MULTIPLE_HEAPS
35211 #ifdef MULTIPLE_HEAPS
35212 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
35214 gc_heap* hp = pGenGCHeap;
35216 // prefix complains about us dereferencing hp in wks build even though we only access static members
35217 // this way. not sure how to shut it up except for this ugly workaround:
35218 PREFIX_ASSUME(hp != NULL);
35220 #endif //MULTIPLE_HEAPS
35222 if (size < loh_size_threshold)
35228 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
35229 #ifdef FEATURE_STRUCTALIGN
35230 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
35231 #endif // FEATURE_STRUCTALIGN
35232 // ASSERT (newAlloc);
35236 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35237 #ifdef FEATURE_STRUCTALIGN
35238 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35239 #endif // FEATURE_STRUCTALIGN
35242 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35245 #ifdef COUNT_CYCLES
35246 finish = GetCycleCount32();
35247 #elif defined(ENABLE_INSTRUMENTATION)
35248 finish = GetInstLogTime();
35249 #endif //COUNT_CYCLES
35250 AllocDuration += finish - AllocStart;
35257 GCHeap::FixAllocContext (gc_alloc_context* context, void* arg, void *heap)
35259 alloc_context* acontext = static_cast<alloc_context*>(context);
35260 #ifdef MULTIPLE_HEAPS
35263 acontext->alloc_count = 0;
35265 uint8_t * alloc_ptr = acontext->alloc_ptr;
35270 // The acontext->alloc_heap can be out of sync with the ptrs because
35271 // of heap re-assignment in allocate
35272 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
35274 gc_heap* hp = pGenGCHeap;
35275 #endif //MULTIPLE_HEAPS
35277 if (heap == NULL || heap == hp)
35279 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
35280 get_alignment_constant(TRUE));
35285 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
35287 uint8_t *o = (uint8_t*)pInteriorPtr;
35289 gc_heap* hp = gc_heap::heap_of (o);
35291 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
35292 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
35294 if (o >= lowest && o < highest)
35296 o = hp->find_object (o, lowest);
35303 return (Object *)o;
35306 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
35308 if (dd_new_allocation (dd) < 0)
35313 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
35321 //----------------------------------------------------------------------------
35322 // #GarbageCollector
35324 // API to ensure that a complete new garbage collection takes place
35327 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
35332 size_t total_allocated = 0;
35333 size_t total_desired = 0;
35334 #ifdef MULTIPLE_HEAPS
35336 for (hn = 0; hn < gc_heap::n_heaps; hn++)
35338 gc_heap* hp = gc_heap::g_heaps [hn];
35339 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
35340 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
35341 dd_new_allocation (hp->dynamic_data_of (0));
35344 gc_heap* hp = pGenGCHeap;
35345 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
35346 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
35347 dd_new_allocation (hp->dynamic_data_of (0));
35348 #endif //MULTIPLE_HEAPS
35350 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
35352 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
35353 total_allocated, total_desired));
35360 #ifdef MULTIPLE_HEAPS
35361 gc_heap* hpt = gc_heap::g_heaps[0];
35364 #endif //MULTIPLE_HEAPS
35366 generation = (generation < 0) ? max_generation : min (generation, max_generation);
35367 dynamic_data* dd = hpt->dynamic_data_of (generation);
35369 #ifdef BACKGROUND_GC
35370 if (recursive_gc_sync::background_running_p())
35372 if ((mode == collection_optimized) || (mode & collection_non_blocking))
35376 if (mode & collection_blocking)
35378 pGenGCHeap->background_gc_wait();
35379 if (mode & collection_optimized)
35385 #endif //BACKGROUND_GC
35387 if (mode & collection_optimized)
35389 if (pGenGCHeap->gc_started)
35395 BOOL should_collect = FALSE;
35396 BOOL should_check_loh = (generation == max_generation);
35397 #ifdef MULTIPLE_HEAPS
35398 for (int i = 0; i < gc_heap::n_heaps; i++)
35400 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
35401 dynamic_data* dd2 = (should_check_loh ?
35402 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
35405 if (should_collect_optimized (dd1, low_memory_p))
35407 should_collect = TRUE;
35410 if (dd2 && should_collect_optimized (dd2, low_memory_p))
35412 should_collect = TRUE;
35417 should_collect = should_collect_optimized (dd, low_memory_p);
35418 if (!should_collect && should_check_loh)
35421 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
35423 #endif //MULTIPLE_HEAPS
35424 if (!should_collect)
35431 size_t CollectionCountAtEntry = dd_collection_count (dd);
35432 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
35433 size_t CurrentCollectionCount = 0;
35437 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
35439 if ((mode & collection_blocking) &&
35440 (generation == max_generation) &&
35441 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
35443 #ifdef BACKGROUND_GC
35444 if (recursive_gc_sync::background_running_p())
35446 pGenGCHeap->background_gc_wait();
35448 #endif //BACKGROUND_GC
35453 if (CollectionCountAtEntry == CurrentCollectionCount)
35462 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
35464 int gen = (generation < 0) ?
35465 max_generation : min (generation, max_generation);
35467 gc_reason reason = reason_empty;
35471 if (mode & collection_blocking)
35473 reason = reason_lowmemory_blocking;
35477 reason = reason_lowmemory;
35482 reason = reason_induced;
35485 if (reason == reason_induced)
35487 if (mode & collection_compacting)
35489 reason = reason_induced_compacting;
35491 else if (mode & collection_non_blocking)
35493 reason = reason_induced_noforce;
35496 else if (mode & collection_gcstress)
35498 reason = reason_gcstress;
35503 return GarbageCollectGeneration (gen, reason);
35506 void gc_heap::do_pre_gc()
35508 STRESS_LOG_GC_STACK;
35511 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
35512 (uint32_t)settings.condemned_generation,
35513 (uint32_t)settings.reason);
35514 #endif // STRESS_LOG
35516 #ifdef MULTIPLE_HEAPS
35517 gc_heap* hp = g_heaps[0];
35520 #endif //MULTIPLE_HEAPS
35522 #ifdef BACKGROUND_GC
35523 settings.b_state = hp->current_bgc_state;
35524 #endif //BACKGROUND_GC
35527 size_t total_allocated_since_last_gc = get_total_allocated_since_last_gc();
35528 #ifdef BACKGROUND_GC
35529 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)(%s)(%d)",
35530 VolatileLoad(&settings.gc_index),
35531 dd_collection_count (hp->dynamic_data_of (0)),
35532 settings.condemned_generation,
35533 total_allocated_since_last_gc,
35534 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
35535 settings.b_state));
35537 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)",
35538 VolatileLoad(&settings.gc_index),
35539 dd_collection_count(hp->dynamic_data_of(0)),
35540 settings.condemned_generation,
35541 total_allocated_since_last_gc));
35542 #endif //BACKGROUND_GC
35544 if (heap_hard_limit)
35546 size_t total_heap_committed = get_total_committed_size();
35547 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35548 dprintf (1, ("(%d)GC commit BEG #%Id: %Id (recorded: %Id)",
35549 settings.condemned_generation,
35550 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded));
35554 // TODO: this can happen...it's because of the way we are calling
35555 // do_pre_gc, will fix later.
35556 //if (last_gc_index > VolatileLoad(&settings.gc_index))
35558 // FATAL_GC_ERROR();
35561 last_gc_index = VolatileLoad(&settings.gc_index);
35562 GCHeap::UpdatePreGCCounters();
35563 #if defined(__linux__)
35564 GCToEEInterface::UpdateGCEventStatus(static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Default)),
35565 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Default)),
35566 static_cast<int>(GCEventStatus::GetEnabledLevel(GCEventProvider_Private)),
35567 static_cast<int>(GCEventStatus::GetEnabledKeywords(GCEventProvider_Private)));
35568 #endif // __linux__
35570 if (settings.concurrent)
35572 #ifdef BACKGROUND_GC
35573 full_gc_counts[gc_type_background]++;
35574 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
35575 GCHeap::gc_stress_fgcs_in_bgc = 0;
35576 #endif // STRESS_HEAP && !FEATURE_REDHAWK
35577 #endif // BACKGROUND_GC
35581 if (settings.condemned_generation == max_generation)
35583 full_gc_counts[gc_type_blocking]++;
35587 #ifdef BACKGROUND_GC
35588 if (settings.background_p)
35590 ephemeral_fgc_counts[settings.condemned_generation]++;
35592 #endif //BACKGROUND_GC
35596 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35597 if (g_fEnableAppDomainMonitoring)
35599 GCToEEInterface::ResetTotalSurvivedBytes();
35601 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35604 #ifdef GC_CONFIG_DRIVEN
35605 void gc_heap::record_interesting_info_per_heap()
35607 // datapoints are always from the last blocking GC so don't record again
35609 if (!(settings.concurrent))
35611 for (int i = 0; i < max_idp_count; i++)
35613 interesting_data_per_heap[i] += interesting_data_per_gc[i];
35617 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
35618 if (compact_reason >= 0)
35619 (compact_reasons_per_heap[compact_reason])++;
35620 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
35621 if (expand_mechanism >= 0)
35622 (expand_mechanisms_per_heap[expand_mechanism])++;
35624 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
35626 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
35627 (interesting_mechanism_bits_per_heap[i])++;
35630 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
35631 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
35633 (size_t)settings.gc_index,
35634 settings.condemned_generation,
35635 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
35636 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
35637 ((expand_mechanism >= 0)? "X" : ""), // EX
35638 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
35639 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
35640 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
35641 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
35642 interesting_data_per_gc[idp_pre_short],
35643 interesting_data_per_gc[idp_post_short],
35644 interesting_data_per_gc[idp_merged_pin],
35645 interesting_data_per_gc[idp_converted_pin],
35646 interesting_data_per_gc[idp_pre_pin],
35647 interesting_data_per_gc[idp_post_pin],
35648 interesting_data_per_gc[idp_pre_and_post_pin],
35649 interesting_data_per_gc[idp_pre_short_padded],
35650 interesting_data_per_gc[idp_post_short_padded]));
35653 void gc_heap::record_global_mechanisms()
35655 for (int i = 0; i < max_global_mechanisms_count; i++)
35657 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
35659 ::record_global_mechanism (i);
35664 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
35666 if (!compact_ratio)
35667 return (!compact_p);
35669 size_t compact_count = compact_or_sweep_gcs[0];
35670 size_t sweep_count = compact_or_sweep_gcs[1];
35672 size_t total_count = compact_count + sweep_count;
35673 BOOL should_compact = compact_p;
35674 if (total_count > 3)
35678 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
35679 if (temp_ratio > compact_ratio)
35681 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
35682 // (compact_count + 1), (total_count + 1), temp_ratio));
35683 should_compact = FALSE;
35688 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
35689 if (temp_ratio > (100 - compact_ratio))
35691 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
35692 // (sweep_count + 1), (total_count + 1), temp_ratio));
35693 should_compact = TRUE;
35698 return !should_compact;
35700 #endif //GC_CONFIG_DRIVEN
35702 bool gc_heap::is_pm_ratio_exceeded()
35704 size_t maxgen_frag = 0;
35705 size_t maxgen_size = 0;
35706 size_t total_heap_size = get_total_heap_size();
35708 #ifdef MULTIPLE_HEAPS
35709 for (int i = 0; i < gc_heap::n_heaps; i++)
35711 gc_heap* hp = gc_heap::g_heaps[i];
35712 #else //MULTIPLE_HEAPS
35714 gc_heap* hp = pGenGCHeap;
35715 #endif //MULTIPLE_HEAPS
35717 maxgen_frag += dd_fragmentation (hp->dynamic_data_of (max_generation));
35718 maxgen_size += hp->generation_size (max_generation);
35721 double maxgen_ratio = (double)maxgen_size / (double)total_heap_size;
35722 double maxgen_frag_ratio = (double)maxgen_frag / (double)maxgen_size;
35723 dprintf (GTC_LOG, ("maxgen %Id(%d%% total heap), frag: %Id (%d%% maxgen)",
35724 maxgen_size, (int)(maxgen_ratio * 100.0),
35725 maxgen_frag, (int)(maxgen_frag_ratio * 100.0)));
35727 bool maxgen_highfrag_p = ((maxgen_ratio > 0.5) && (maxgen_frag_ratio > 0.1));
35729 // We need to adjust elevation here because if there's enough fragmentation it's not
35731 if (maxgen_highfrag_p)
35733 settings.should_lock_elevation = FALSE;
35734 dprintf (GTC_LOG, ("high frag gen2, turn off elevation"));
35737 return maxgen_highfrag_p;
35740 void gc_heap::do_post_gc()
35742 if (!settings.concurrent)
35748 #ifdef COUNT_CYCLES
35749 AllocStart = GetCycleCount32();
35751 AllocStart = clock();
35752 #endif //COUNT_CYCLES
35755 #ifdef MULTIPLE_HEAPS
35756 gc_heap* hp = g_heaps[0];
35759 #endif //MULTIPLE_HEAPS
35761 GCToEEInterface::GcDone(settings.condemned_generation);
35763 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
35764 (uint32_t)settings.condemned_generation,
35765 (uint32_t)settings.reason,
35766 !!settings.concurrent);
35768 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
35769 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
35770 VolatileLoad(&settings.gc_index),
35771 dd_collection_count(hp->dynamic_data_of(0)),
35772 settings.condemned_generation,
35773 (settings.concurrent ? "BGC" : "GC")));
35775 if (settings.exit_memory_load != 0)
35776 last_gc_memory_load = settings.exit_memory_load;
35777 else if (settings.entry_memory_load != 0)
35778 last_gc_memory_load = settings.entry_memory_load;
35780 last_gc_heap_size = get_total_heap_size();
35781 last_gc_fragmentation = get_total_fragmentation();
35784 if (heap_hard_limit)
35786 size_t total_heap_committed = get_total_committed_size();
35787 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35788 dprintf (1, ("(%d)GC commit END #%Id: %Id (recorded: %Id), heap %Id, frag: %Id",
35789 settings.condemned_generation,
35790 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded,
35791 last_gc_heap_size, last_gc_fragmentation));
35795 // Note we only do this at the end of full blocking GCs because we do not want
35796 // to turn on this provisional mode during the middle of a BGC.
35797 if ((settings.condemned_generation == max_generation) && (!settings.concurrent))
35801 size_t full_compacting_gc_count = full_gc_counts[gc_type_compacting];
35802 if (provisional_mode_triggered)
35804 uint64_t r = gc_rand::get_rand(10);
35805 if ((full_compacting_gc_count - provisional_triggered_gc_count) >= r)
35807 provisional_mode_triggered = false;
35808 provisional_off_gc_count = full_compacting_gc_count;
35809 dprintf (GTC_LOG, ("%Id NGC2s when turned on, %Id NGCs since(%Id)",
35810 provisional_triggered_gc_count, (full_compacting_gc_count - provisional_triggered_gc_count),
35811 num_provisional_triggered));
35816 uint64_t r = gc_rand::get_rand(5);
35817 if ((full_compacting_gc_count - provisional_off_gc_count) >= r)
35819 provisional_mode_triggered = true;
35820 provisional_triggered_gc_count = full_compacting_gc_count;
35821 num_provisional_triggered++;
35822 dprintf (GTC_LOG, ("%Id NGC2s when turned off, %Id NGCs since(%Id)",
35823 provisional_off_gc_count, (full_compacting_gc_count - provisional_off_gc_count),
35824 num_provisional_triggered));
35830 if (provisional_mode_triggered)
35832 if ((settings.entry_memory_load < high_memory_load_th) ||
35833 !is_pm_ratio_exceeded())
35835 dprintf (GTC_LOG, ("turning off PM"));
35836 provisional_mode_triggered = false;
35839 else if ((settings.entry_memory_load >= high_memory_load_th) && is_pm_ratio_exceeded())
35841 dprintf (GTC_LOG, ("highmem && highfrag - turning on PM"));
35842 provisional_mode_triggered = true;
35843 num_provisional_triggered++;
35848 GCHeap::UpdatePostGCCounters();
35849 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35850 //if (g_fEnableARM)
35852 // SystemDomain::GetADSurvivedBytes();
35854 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35857 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
35858 (uint32_t)settings.condemned_generation,
35859 (uint32_t)settings.reason);
35860 #endif // STRESS_LOG
35862 #ifdef GC_CONFIG_DRIVEN
35863 if (!settings.concurrent)
35865 if (settings.compaction)
35866 (compact_or_sweep_gcs[0])++;
35868 (compact_or_sweep_gcs[1])++;
35871 #ifdef MULTIPLE_HEAPS
35872 for (int i = 0; i < n_heaps; i++)
35873 g_heaps[i]->record_interesting_info_per_heap();
35875 record_interesting_info_per_heap();
35876 #endif //MULTIPLE_HEAPS
35877 record_global_mechanisms();
35878 #endif //GC_CONFIG_DRIVEN
35881 unsigned GCHeap::GetGcCount()
35883 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
35887 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
35889 dprintf (2, ("triggered a GC!"));
35891 #ifdef MULTIPLE_HEAPS
35892 gc_heap* hpt = gc_heap::g_heaps[0];
35895 #endif //MULTIPLE_HEAPS
35896 bool cooperative_mode = true;
35897 dynamic_data* dd = hpt->dynamic_data_of (gen);
35898 size_t localCount = dd_collection_count (dd);
35900 enter_spin_lock (&gc_heap::gc_lock);
35901 dprintf (SPINLOCK_LOG, ("GC Egc"));
35902 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
35904 //don't trigger another GC if one was already in progress
35905 //while waiting for the lock
35907 size_t col_count = dd_collection_count (dd);
35909 if (localCount != col_count)
35911 #ifdef SYNCHRONIZATION_STATS
35912 gc_lock_contended++;
35913 #endif //SYNCHRONIZATION_STATS
35914 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
35915 leave_spin_lock (&gc_heap::gc_lock);
35917 // We don't need to release msl here 'cause this means a GC
35918 // has happened and would have release all msl's.
35923 #ifdef COUNT_CYCLES
35924 int gc_start = GetCycleCount32();
35925 #endif //COUNT_CYCLES
35928 #ifdef COUNT_CYCLES
35929 AllocDuration += GetCycleCount32() - AllocStart;
35931 AllocDuration += clock() - AllocStart;
35932 #endif //COUNT_CYCLES
35935 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
35936 (reason == reason_lowmemory_blocking) ||
35937 (gc_heap::latency_level == latency_level_memory_footprint);
35939 gc_trigger_reason = reason;
35941 #ifdef MULTIPLE_HEAPS
35942 for (int i = 0; i < gc_heap::n_heaps; i++)
35944 gc_heap::g_heaps[i]->reset_gc_done();
35947 gc_heap::reset_gc_done();
35948 #endif //MULTIPLE_HEAPS
35950 gc_heap::gc_started = TRUE;
35953 init_sync_log_stats();
35955 #ifndef MULTIPLE_HEAPS
35956 cooperative_mode = gc_heap::enable_preemptive ();
35958 dprintf (2, ("Suspending EE"));
35959 BEGIN_TIMING(suspend_ee_during_log);
35960 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
35961 END_TIMING(suspend_ee_during_log);
35962 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
35963 gc_heap::disable_preemptive (cooperative_mode);
35964 if (gc_heap::proceed_with_gc_p)
35965 pGenGCHeap->settings.init_mechanisms();
35967 gc_heap::update_collection_counts_for_no_gc();
35969 #endif //!MULTIPLE_HEAPS
35972 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
35975 #ifdef COUNT_CYCLES
35978 start = GetCycleCount32();
35983 #endif //COUNT_CYCLES
35984 PromotedObjectCount = 0;
35987 unsigned int condemned_generation_number = gen;
35989 // We want to get a stack from the user thread that triggered the GC
35990 // instead of on the GC thread which is the case for Server GC.
35991 // But we are doing it for Workstation GC as well to be uniform.
35992 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35994 #ifdef MULTIPLE_HEAPS
35995 GcCondemnedGeneration = condemned_generation_number;
35997 cooperative_mode = gc_heap::enable_preemptive ();
35999 BEGIN_TIMING(gc_during_log);
36000 gc_heap::ee_suspend_event.Set();
36001 gc_heap::wait_for_gc_done();
36002 END_TIMING(gc_during_log);
36004 gc_heap::disable_preemptive (cooperative_mode);
36006 condemned_generation_number = GcCondemnedGeneration;
36008 if (gc_heap::proceed_with_gc_p)
36010 BEGIN_TIMING(gc_during_log);
36011 pGenGCHeap->garbage_collect (condemned_generation_number);
36012 if (gc_heap::pm_trigger_full_gc)
36014 pGenGCHeap->garbage_collect_pm_full_gc();
36016 END_TIMING(gc_during_log);
36018 #endif //MULTIPLE_HEAPS
36021 #ifdef COUNT_CYCLES
36022 finish = GetCycleCount32();
36025 #endif //COUNT_CYCLES
36026 GcDuration += finish - start;
36028 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
36029 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
36030 finish - start, GcDuration,
36031 AllocCount ? (AllocDuration / AllocCount) : 0,
36032 AllocSmallCount, AllocBigCount));
36037 #ifdef BACKGROUND_GC
36038 // We are deciding whether we should fire the alloc wait end event here
36039 // because in begin_foreground we could be calling end_foreground
36040 // if we need to retry.
36041 if (gc_heap::alloc_wait_event_p)
36043 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
36044 gc_heap::alloc_wait_event_p = FALSE;
36046 #endif //BACKGROUND_GC
36048 #ifndef MULTIPLE_HEAPS
36049 #ifdef BACKGROUND_GC
36050 if (!gc_heap::dont_restart_ee_p)
36052 #endif //BACKGROUND_GC
36053 BEGIN_TIMING(restart_ee_during_log);
36054 GCToEEInterface::RestartEE(TRUE);
36055 END_TIMING(restart_ee_during_log);
36056 #ifdef BACKGROUND_GC
36058 #endif //BACKGROUND_GC
36059 #endif //!MULTIPLE_HEAPS
36061 #ifdef COUNT_CYCLES
36062 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
36063 GetCycleCount32() - gc_start);
36064 #endif //COUNT_CYCLES
36066 #ifndef MULTIPLE_HEAPS
36067 process_sync_log_stats();
36068 gc_heap::gc_started = FALSE;
36069 gc_heap::set_gc_done();
36070 dprintf (SPINLOCK_LOG, ("GC Lgc"));
36071 leave_spin_lock (&gc_heap::gc_lock);
36072 #endif //!MULTIPLE_HEAPS
36074 #ifdef FEATURE_PREMORTEM_FINALIZATION
36075 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
36076 #endif // FEATURE_PREMORTEM_FINALIZATION
36078 return dd_collection_count (dd);
36081 size_t GCHeap::GetTotalBytesInUse ()
36083 #ifdef MULTIPLE_HEAPS
36084 //enumarate all the heaps and get their size.
36085 size_t tot_size = 0;
36086 for (int i = 0; i < gc_heap::n_heaps; i++)
36088 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
36089 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
36093 return ApproxTotalBytesInUse ();
36094 #endif //MULTIPLE_HEAPS
36097 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
36099 if (get_bgc_fgc_count != 0)
36101 #ifdef BACKGROUND_GC
36102 if (generation == max_generation)
36104 return (int)(gc_heap::full_gc_counts[gc_type_background]);
36108 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
36112 #endif //BACKGROUND_GC
36115 #ifdef MULTIPLE_HEAPS
36116 gc_heap* hp = gc_heap::g_heaps [0];
36117 #else //MULTIPLE_HEAPS
36118 gc_heap* hp = pGenGCHeap;
36119 #endif //MULTIPLE_HEAPS
36120 if (generation > max_generation)
36123 return (int)dd_collection_count (hp->dynamic_data_of (generation));
36126 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
36128 size_t totsize = 0;
36130 //ASSERT(InMustComplete());
36131 enter_spin_lock (&pGenGCHeap->gc_lock);
36133 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
36134 // Get small block heap size info
36135 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
36136 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
36137 while (seg1 != eph_seg)
36139 totsize += heap_segment_allocated (seg1) -
36140 heap_segment_mem (seg1);
36141 seg1 = heap_segment_next (seg1);
36144 //discount the fragmentation
36145 for (int i = 0; i <= max_generation; i++)
36147 generation* gen = pGenGCHeap->generation_of (i);
36148 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
36151 if (!small_heap_only)
36153 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
36157 totsize += heap_segment_allocated (seg2) -
36158 heap_segment_mem (seg2);
36159 seg2 = heap_segment_next (seg2);
36162 //discount the fragmentation
36163 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
36164 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
36167 leave_spin_lock (&pGenGCHeap->gc_lock);
36171 #ifdef MULTIPLE_HEAPS
36172 void GCHeap::AssignHeap (alloc_context* acontext)
36174 // Assign heap based on processor
36175 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
36176 acontext->set_home_heap(acontext->get_alloc_heap());
36178 GCHeap* GCHeap::GetHeap (int n)
36180 assert (n < gc_heap::n_heaps);
36181 return gc_heap::g_heaps [n]->vm_heap;
36183 #endif //MULTIPLE_HEAPS
36185 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
36187 alloc_context* acontext = static_cast<alloc_context*>(context);
36188 #ifdef MULTIPLE_HEAPS
36189 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
36190 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
36192 UNREFERENCED_PARAMETER(acontext);
36193 UNREFERENCED_PARAMETER(thread_number);
36195 #endif //MULTIPLE_HEAPS
36198 // Returns the number of processors required to trigger the use of thread based allocation contexts
36199 int GCHeap::GetNumberOfHeaps ()
36201 #ifdef MULTIPLE_HEAPS
36202 return gc_heap::n_heaps;
36205 #endif //MULTIPLE_HEAPS
36209 in this way we spend extra time cycling through all the heaps while create the handle
36210 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
36212 int GCHeap::GetHomeHeapNumber ()
36214 #ifdef MULTIPLE_HEAPS
36215 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
36221 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
36222 return (hp ? hp->pGenGCHeap->heap_number : 0);
36225 #endif //MULTIPLE_HEAPS
36228 unsigned int GCHeap::GetCondemnedGeneration()
36230 return gc_heap::settings.condemned_generation;
36233 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
36234 uint64_t* totalPhysicalMem,
36235 uint32_t* lastRecordedMemLoad,
36236 size_t* lastRecordedHeapSize,
36237 size_t* lastRecordedFragmentation)
36239 *highMemLoadThreshold = gc_heap::high_memory_load_th;
36240 *totalPhysicalMem = gc_heap::total_physical_mem;
36241 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
36242 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
36243 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
36246 int GCHeap::GetGcLatencyMode()
36248 return (int)(pGenGCHeap->settings.pause_mode);
36251 int GCHeap::SetGcLatencyMode (int newLatencyMode)
36253 if (gc_heap::settings.pause_mode == pause_no_gc)
36254 return (int)set_pause_mode_no_gc;
36256 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
36258 if (new_mode == pause_low_latency)
36260 #ifndef MULTIPLE_HEAPS
36261 pGenGCHeap->settings.pause_mode = new_mode;
36262 #endif //!MULTIPLE_HEAPS
36264 else if (new_mode == pause_sustained_low_latency)
36266 #ifdef BACKGROUND_GC
36267 if (gc_heap::gc_can_use_concurrent)
36269 pGenGCHeap->settings.pause_mode = new_mode;
36271 #endif //BACKGROUND_GC
36275 pGenGCHeap->settings.pause_mode = new_mode;
36278 #ifdef BACKGROUND_GC
36279 if (recursive_gc_sync::background_running_p())
36281 // If we get here, it means we are doing an FGC. If the pause
36282 // mode was altered we will need to save it in the BGC settings.
36283 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
36285 gc_heap::saved_bgc_settings.pause_mode = new_mode;
36288 #endif //BACKGROUND_GC
36290 return (int)set_pause_mode_success;
36293 int GCHeap::GetLOHCompactionMode()
36295 return pGenGCHeap->loh_compaction_mode;
36298 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
36300 #ifdef FEATURE_LOH_COMPACTION
36301 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
36302 #endif //FEATURE_LOH_COMPACTION
36305 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
36306 uint32_t lohPercentage)
36308 #ifdef MULTIPLE_HEAPS
36309 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36311 gc_heap* hp = gc_heap::g_heaps [hn];
36312 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
36314 #else //MULTIPLE_HEAPS
36315 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
36316 #endif //MULTIPLE_HEAPS
36318 pGenGCHeap->full_gc_approach_event.Reset();
36319 pGenGCHeap->full_gc_end_event.Reset();
36320 pGenGCHeap->full_gc_approach_event_set = false;
36322 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
36323 pGenGCHeap->fgn_loh_percent = lohPercentage;
36328 bool GCHeap::CancelFullGCNotification()
36330 pGenGCHeap->fgn_maxgen_percent = 0;
36331 pGenGCHeap->fgn_loh_percent = 0;
36333 pGenGCHeap->full_gc_approach_event.Set();
36334 pGenGCHeap->full_gc_end_event.Set();
36339 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
36341 dprintf (2, ("WFGA: Begin wait"));
36342 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
36343 dprintf (2, ("WFGA: End wait"));
36347 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
36349 dprintf (2, ("WFGE: Begin wait"));
36350 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
36351 dprintf (2, ("WFGE: End wait"));
36355 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
36357 NoGCRegionLockHolder lh;
36359 dprintf (1, ("begin no gc called"));
36360 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
36361 if (status == start_no_gc_success)
36363 GarbageCollect (max_generation);
36364 status = gc_heap::get_start_no_gc_region_status();
36367 if (status != start_no_gc_success)
36368 gc_heap::handle_failure_for_no_gc();
36370 return (int)status;
36373 int GCHeap::EndNoGCRegion()
36375 NoGCRegionLockHolder lh;
36376 return (int)gc_heap::end_no_gc_region();
36379 void GCHeap::PublishObject (uint8_t* Obj)
36381 #ifdef BACKGROUND_GC
36382 gc_heap* hp = gc_heap::heap_of (Obj);
36383 hp->bgc_alloc_lock->loh_alloc_done (Obj);
36384 hp->bgc_untrack_loh_alloc();
36385 #endif //BACKGROUND_GC
36388 // The spec for this one isn't clear. This function
36389 // returns the size that can be allocated without
36390 // triggering a GC of any kind.
36391 size_t GCHeap::ApproxFreeBytes()
36394 //ASSERT(InMustComplete());
36395 enter_spin_lock (&pGenGCHeap->gc_lock);
36397 generation* gen = pGenGCHeap->generation_of (0);
36398 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
36400 leave_spin_lock (&pGenGCHeap->gc_lock);
36405 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
36407 if ((gen < 0) || (gen > max_generation))
36409 #ifdef MULTIPLE_HEAPS
36410 counters->current_size = 0;
36411 counters->promoted_size = 0;
36412 counters->collection_count = 0;
36414 //enumarate all the heaps and get their counters.
36415 for (int i = 0; i < gc_heap::n_heaps; i++)
36417 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
36419 counters->current_size += dd_current_size (dd);
36420 counters->promoted_size += dd_promoted_size (dd);
36422 counters->collection_count += dd_collection_count (dd);
36425 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
36426 counters->current_size = dd_current_size (dd);
36427 counters->promoted_size = dd_promoted_size (dd);
36428 counters->collection_count = dd_collection_count (dd);
36429 #endif //MULTIPLE_HEAPS
36433 // Get the segment size to use, making sure it conforms.
36434 size_t GCHeap::GetValidSegmentSize(bool large_seg)
36436 return (large_seg ? gc_heap::min_loh_segment_size : gc_heap::soh_segment_size);
36439 // Get the max gen0 heap size, making sure it conforms.
36440 size_t gc_heap::get_gen0_min_size()
36442 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
36443 bool is_config_invalid = ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size));
36444 if (is_config_invalid)
36447 // performance data seems to indicate halving the size results
36448 // in optimal perf. Ask for adjusted gen0 size.
36449 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
36451 // if gen0 size is too large given the available memory, reduce it.
36452 // Get true cache size, as we don't want to reduce below this.
36453 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
36454 dprintf (1, ("cache: %Id-%Id",
36455 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
36456 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
36458 int n_heaps = gc_heap::n_heaps;
36460 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
36461 gen0size = max((4*trueSize/5),(256*1024));
36462 trueSize = max(trueSize, (256*1024));
36466 dprintf (1, ("gen0size: %Id * %d = %Id, physical mem: %Id / 6 = %Id",
36467 gen0size, n_heaps, (gen0size * n_heaps),
36468 gc_heap::total_physical_mem,
36469 gc_heap::total_physical_mem / 6));
36471 // if the total min GC across heaps will exceed 1/6th of available memory,
36472 // then reduce the min GC size until it either fits or has been reduced to cache size.
36473 while ((gen0size * n_heaps) > (gc_heap::total_physical_mem / 6))
36475 gen0size = gen0size / 2;
36476 if (gen0size <= trueSize)
36478 gen0size = trueSize;
36484 size_t seg_size = gc_heap::soh_segment_size;
36487 // Generation 0 must never be more than 1/2 the segment size.
36488 if (gen0size >= (seg_size / 2))
36489 gen0size = seg_size / 2;
36491 // If the value from config is valid we use it as is without this adjustment.
36492 if (is_config_invalid)
36494 if (heap_hard_limit)
36496 size_t gen0size_seg = seg_size / 8;
36497 if (gen0size >= gen0size_seg)
36499 dprintf (1, ("gen0 limited by seg size %Id->%Id", gen0size, gen0size_seg));
36500 gen0size = gen0size_seg;
36504 gen0size = gen0size / 8 * 5;
36507 gen0size = Align (gen0size);
36512 void GCHeap::SetReservedVMLimit (size_t vmlimit)
36514 gc_heap::reserved_memory_limit = vmlimit;
36517 //versions of same method on each heap
36519 #ifdef FEATURE_PREMORTEM_FINALIZATION
36521 Object* GCHeap::GetNextFinalizableObject()
36524 #ifdef MULTIPLE_HEAPS
36526 //return the first non critical one in the first queue.
36527 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36529 gc_heap* hp = gc_heap::g_heaps [hn];
36530 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
36534 //return the first non crtitical/critical one in the first queue.
36535 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36537 gc_heap* hp = gc_heap::g_heaps [hn];
36538 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
36545 #else //MULTIPLE_HEAPS
36546 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
36547 #endif //MULTIPLE_HEAPS
36551 size_t GCHeap::GetNumberFinalizableObjects()
36553 #ifdef MULTIPLE_HEAPS
36555 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36557 gc_heap* hp = gc_heap::g_heaps [hn];
36558 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
36563 #else //MULTIPLE_HEAPS
36564 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
36565 #endif //MULTIPLE_HEAPS
36568 size_t GCHeap::GetFinalizablePromotedCount()
36570 #ifdef MULTIPLE_HEAPS
36573 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36575 gc_heap* hp = gc_heap::g_heaps [hn];
36576 cnt += hp->finalize_queue->GetPromotedCount();
36580 #else //MULTIPLE_HEAPS
36581 return pGenGCHeap->finalize_queue->GetPromotedCount();
36582 #endif //MULTIPLE_HEAPS
36585 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
36587 #ifdef MULTIPLE_HEAPS
36588 bool foundp = false;
36589 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36591 gc_heap* hp = gc_heap::g_heaps [hn];
36592 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
36597 #else //MULTIPLE_HEAPS
36598 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
36599 #endif //MULTIPLE_HEAPS
36602 bool GCHeap::ShouldRestartFinalizerWatchDog()
36604 // This condition was historically used as part of the condition to detect finalizer thread timeouts
36605 return gc_heap::gc_lock.lock != -1;
36608 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
36610 #ifdef MULTIPLE_HEAPS
36611 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36613 gc_heap* hp = gc_heap::g_heaps [hn];
36614 hp->finalize_queue->SetSegForShutDown(fHasLock);
36617 #else //MULTIPLE_HEAPS
36618 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
36619 #endif //MULTIPLE_HEAPS
36622 //---------------------------------------------------------------------------
36623 // Finalized class tracking
36624 //---------------------------------------------------------------------------
36626 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
36630 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
36632 //just reset the bit
36633 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
36638 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
36639 return hp->finalize_queue->RegisterForFinalization (gen, obj);
36643 void GCHeap::SetFinalizationRun (Object* obj)
36645 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
36649 //--------------------------------------------------------------------
36651 // Support for finalization
36653 //--------------------------------------------------------------------
36656 unsigned int gen_segment (int gen)
36658 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
36659 return (NUMBERGENERATIONS - gen - 1);
36662 bool CFinalize::Initialize()
36669 m_Array = new (nothrow)(Object*[100]);
36674 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
36675 if (GCConfig::GetBreakOnOOM())
36677 GCToOSInterface::DebugBreak();
36681 m_EndArray = &m_Array[100];
36683 for (int i =0; i < FreeList; i++)
36685 SegQueueLimit (i) = m_Array;
36687 m_PromotedCount = 0;
36690 lockowner_threadid.Clear();
36696 CFinalize::~CFinalize()
36701 size_t CFinalize::GetPromotedCount ()
36703 return m_PromotedCount;
36707 void CFinalize::EnterFinalizeLock()
36709 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36710 GCToEEInterface::GetThread() == 0 ||
36711 GCToEEInterface::IsPreemptiveGCDisabled());
36714 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
36716 unsigned int i = 0;
36719 YieldProcessor(); // indicate to the processor that we are spinning
36721 GCToOSInterface::YieldThread (0);
36723 GCToOSInterface::Sleep (5);
36729 lockowner_threadid.SetToCurrentThread();
36734 void CFinalize::LeaveFinalizeLock()
36736 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36737 GCToEEInterface::GetThread() == 0 ||
36738 GCToEEInterface::IsPreemptiveGCDisabled());
36741 lockowner_threadid.Clear();
36747 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
36754 EnterFinalizeLock();
36756 unsigned int dest = 0;
36758 if (g_fFinalizerRunOnShutDown)
36760 //no method table available yet,
36761 //put it in the finalizer queue and sort out when
36763 dest = FinalizerListSeg;
36767 dest = gen_segment (gen);
36769 // Adjust boundary for segments so that GC will keep objects alive.
36770 Object*** s_i = &SegQueue (FreeList);
36771 if ((*s_i) == m_EndArray)
36775 LeaveFinalizeLock();
36776 if (method_table(obj) == NULL)
36778 // If the object is uninitialized, a valid size should have been passed.
36779 assert (size >= Align (min_obj_size));
36780 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
36781 ((CObjectHeader*)obj)->SetFree(size);
36783 STRESS_LOG_OOM_STACK(0);
36784 if (GCConfig::GetBreakOnOOM())
36786 GCToOSInterface::DebugBreak();
36791 Object*** end_si = &SegQueueLimit (dest);
36794 //is the segment empty?
36795 if (!(*s_i == *(s_i-1)))
36797 //no, swap the end elements.
36798 *(*s_i) = *(*(s_i-1));
36800 //increment the fill pointer
36802 //go to the next segment.
36804 } while (s_i > end_si);
36806 // We have reached the destination segment
36807 // store the object
36809 // increment the fill pointer
36812 LeaveFinalizeLock();
36818 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
36822 EnterFinalizeLock();
36825 if (!IsSegEmpty(FinalizerListSeg))
36827 if (g_fFinalizerRunOnShutDown)
36829 obj = *(SegQueueLimit (FinalizerListSeg)-1);
36830 if (method_table(obj)->HasCriticalFinalizer())
36832 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
36833 FinalizerListSeg, CriticalFinalizerListSeg);
36837 --SegQueueLimit (FinalizerListSeg);
36840 obj = *(--SegQueueLimit (FinalizerListSeg));
36843 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
36845 //the FinalizerList is empty, we can adjust both
36846 // limit instead of moving the object to the free list
36847 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
36848 --SegQueueLimit (FinalizerListSeg);
36852 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
36854 LeaveFinalizeLock();
36859 CFinalize::SetSegForShutDown(BOOL fHasLock)
36864 EnterFinalizeLock();
36865 for (i = 0; i <= max_generation; i++)
36867 unsigned int seg = gen_segment (i);
36868 Object** startIndex = SegQueueLimit (seg)-1;
36869 Object** stopIndex = SegQueue (seg);
36870 for (Object** po = startIndex; po >= stopIndex; po--)
36873 if (method_table(obj)->HasCriticalFinalizer())
36875 MoveItem (po, seg, CriticalFinalizerListSeg);
36879 MoveItem (po, seg, FinalizerListSeg);
36884 LeaveFinalizeLock();
36888 CFinalize::DiscardNonCriticalObjects()
36890 //empty the finalization queue
36891 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
36892 Object** stopIndex = SegQueue (FinalizerListSeg);
36893 for (Object** po = startIndex; po >= stopIndex; po--)
36895 MoveItem (po, FinalizerListSeg, FreeList);
36900 CFinalize::GetNumberFinalizableObjects()
36902 return SegQueueLimit (FinalizerListSeg) -
36903 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
36907 CFinalize::FinalizeSegForAppDomain (void *pDomain,
36908 BOOL fRunFinalizers,
36911 BOOL finalizedFound = FALSE;
36912 Object** endIndex = SegQueue (Seg);
36913 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36915 CObjectHeader* obj = (CObjectHeader*)*i;
36917 // Objects are put into the finalization queue before they are complete (ie their methodtable
36918 // may be null) so we must check that the object we found has a method table before checking
36919 // if it has the index we are looking for. If the methodtable is null, it can't be from the
36920 // unloading domain, so skip it.
36921 if (method_table(obj) == NULL)
36926 // does the EE actually want us to finalize this object?
36927 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
36932 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36934 //remove the object because we don't want to
36935 //run the finalizer
36936 MoveItem (i, Seg, FreeList);
36937 //Reset the bit so it will be put back on the queue
36938 //if resurrected and re-registered.
36939 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36943 if (method_table(obj)->HasCriticalFinalizer())
36945 finalizedFound = TRUE;
36946 MoveItem (i, Seg, CriticalFinalizerListSeg);
36950 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
36952 MoveItem (i, Seg, FreeList);
36956 finalizedFound = TRUE;
36957 MoveItem (i, Seg, FinalizerListSeg);
36963 return finalizedFound;
36967 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
36969 bool finalizedFound = false;
36971 unsigned int startSeg = gen_segment (max_generation);
36973 EnterFinalizeLock();
36975 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
36977 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
36979 finalizedFound = true;
36983 LeaveFinalizeLock();
36985 return finalizedFound;
36989 CFinalize::MoveItem (Object** fromIndex,
36990 unsigned int fromSeg,
36991 unsigned int toSeg)
36995 ASSERT (fromSeg != toSeg);
36996 if (fromSeg > toSeg)
37000 // Place the element at the boundary closest to dest
37001 Object** srcIndex = fromIndex;
37002 for (unsigned int i = fromSeg; i != toSeg; i+= step)
37004 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
37005 Object** destIndex = destFill - (step + 1)/2;
37006 if (srcIndex != destIndex)
37008 Object* tmp = *srcIndex;
37009 *srcIndex = *destIndex;
37013 srcIndex = destIndex;
37018 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
37024 pSC->thread_number = hn;
37026 //scan the finalization queue
37027 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
37028 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
37030 for (Object** po = startIndex; po < stopIndex; po++)
37033 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
37034 dprintf (3, ("scan f %Ix", (size_t)o));
37035 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
37036 if (g_fEnableAppDomainMonitoring)
37038 pSC->pCurrentDomain = GCToEEInterface::GetAppDomainForObject(o);
37040 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
37046 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
37048 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
37049 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
37050 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
37051 for (Object** po = startIndex; po < stopIndex; po++)
37054 fn(po < stopCriticalIndex, *po);
37059 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
37063 sc.promotion = TRUE;
37064 #ifdef MULTIPLE_HEAPS
37065 sc.thread_number = hp->heap_number;
37067 UNREFERENCED_PARAMETER(hp);
37068 #endif //MULTIPLE_HEAPS
37070 BOOL finalizedFound = FALSE;
37072 //start with gen and explore all the younger generations.
37073 unsigned int startSeg = gen_segment (gen);
37075 m_PromotedCount = 0;
37076 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
37078 Object** endIndex = SegQueue (Seg);
37079 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
37081 CObjectHeader* obj = (CObjectHeader*)*i;
37082 dprintf (3, ("scanning: %Ix", (size_t)obj));
37083 if (!g_theGCHeap->IsPromoted (obj))
37085 dprintf (3, ("freacheable: %Ix", (size_t)obj));
37087 assert (method_table(obj)->HasFinalizer());
37089 if (GCToEEInterface::EagerFinalized(obj))
37091 MoveItem (i, Seg, FreeList);
37093 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
37095 //remove the object because we don't want to
37096 //run the finalizer
37098 MoveItem (i, Seg, FreeList);
37100 //Reset the bit so it will be put back on the queue
37101 //if resurrected and re-registered.
37102 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
37109 if (method_table(obj)->HasCriticalFinalizer())
37111 MoveItem (i, Seg, CriticalFinalizerListSeg);
37115 MoveItem (i, Seg, FinalizerListSeg);
37119 #ifdef BACKGROUND_GC
37122 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
37124 // TODO - fix the following line.
37125 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
37126 dprintf (3, ("%Ix is marked", (size_t)obj));
37129 #endif //BACKGROUND_GC
37133 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
37134 !IsSegEmpty(CriticalFinalizerListSeg);
37136 if (finalizedFound)
37138 //Promote the f-reachable objects
37140 #ifdef MULTIPLE_HEAPS
37144 #endif //MULTIPLE_HEAPS
37147 hp->settings.found_finalizers = TRUE;
37149 #ifdef BACKGROUND_GC
37150 if (hp->settings.concurrent)
37152 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
37154 #endif //BACKGROUND_GC
37155 if (hp->settings.concurrent && hp->settings.found_finalizers)
37158 GCToEEInterface::EnableFinalization(true);
37162 return finalizedFound;
37165 //Relocates all of the objects in the finalization array
37167 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
37170 sc.promotion = FALSE;
37171 #ifdef MULTIPLE_HEAPS
37172 sc.thread_number = hp->heap_number;
37174 UNREFERENCED_PARAMETER(hp);
37175 #endif //MULTIPLE_HEAPS
37177 unsigned int Seg = gen_segment (gen);
37179 Object** startIndex = SegQueue (Seg);
37180 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
37182 GCHeap::Relocate (po, &sc);
37187 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
37189 // update the generation fill pointers.
37190 // if gen_0_empty is FALSE, test each object to find out if
37191 // it was promoted or not
37194 for (int i = min (gen+1, max_generation); i > 0; i--)
37196 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
37201 //Look for demoted or promoted plugs
37203 for (int i = gen; i >= 0; i--)
37205 unsigned int Seg = gen_segment (i);
37206 Object** startIndex = SegQueue (Seg);
37208 for (Object** po = startIndex;
37209 po < SegQueueLimit (gen_segment(i)); po++)
37211 int new_gen = g_theGCHeap->WhichGeneration (*po);
37217 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37222 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37223 //back down in order to see all objects.
37234 CFinalize::GrowArray()
37236 size_t oldArraySize = (m_EndArray - m_Array);
37237 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
37239 Object** newArray = new (nothrow) Object*[newArraySize];
37242 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
37243 // to throw for us.
37244 // ASSERT (newArray);
37247 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
37249 //adjust the fill pointers
37250 for (int i = 0; i < FreeList; i++)
37252 m_FillPointers [i] += (newArray - m_Array);
37255 m_Array = newArray;
37256 m_EndArray = &m_Array [newArraySize];
37262 void CFinalize::CheckFinalizerObjects()
37264 for (int i = 0; i <= max_generation; i++)
37266 Object **startIndex = SegQueue (gen_segment (i));
37267 Object **stopIndex = SegQueueLimit (gen_segment (i));
37269 for (Object **po = startIndex; po < stopIndex; po++)
37271 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
37273 ((CObjectHeader*)*po)->Validate();
37277 #endif //VERIFY_HEAP
37279 #endif // FEATURE_PREMORTEM_FINALIZATION
37282 //------------------------------------------------------------------------------
37284 // End of VM specific support
37286 //------------------------------------------------------------------------------
37287 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37289 generation* gen = gc_heap::generation_of (gen_number);
37290 heap_segment* seg = generation_start_segment (gen);
37291 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
37292 generation_allocation_start (gen));
37294 uint8_t* end = heap_segment_allocated (seg);
37295 BOOL small_object_segments = TRUE;
37296 int align_const = get_alignment_constant (small_object_segments);
37303 if ((seg = heap_segment_next (seg)) != 0)
37305 x = heap_segment_mem (seg);
37306 end = heap_segment_allocated (seg);
37311 if (small_object_segments && walk_large_object_heap_p)
37314 small_object_segments = FALSE;
37315 align_const = get_alignment_constant (small_object_segments);
37316 seg = generation_start_segment (large_object_generation);
37317 x = heap_segment_mem (seg);
37318 end = heap_segment_allocated (seg);
37328 size_t s = size (x);
37329 CObjectHeader* o = (CObjectHeader*)x;
37334 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
37336 if (!fn (o->GetObjectBase(), context))
37339 x = x + Align (s, align_const);
37343 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
37345 #ifdef FEATURE_PREMORTEM_FINALIZATION
37346 finalize_queue->WalkFReachableObjects (fn);
37347 #endif //FEATURE_PREMORTEM_FINALIZATION
37350 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37352 #ifdef MULTIPLE_HEAPS
37353 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37355 gc_heap* hp = gc_heap::g_heaps [hn];
37357 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
37360 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
37361 #endif //MULTIPLE_HEAPS
37364 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
37366 uint8_t* o = (uint8_t*)obj;
37369 go_through_object_cl (method_table (o), o, size(o), oo,
37373 Object *oh = (Object*)*oo;
37374 if (!fn (oh, context))
37382 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
37384 gc_heap* hp = (gc_heap*)gc_context;
37385 hp->walk_survivors (fn, diag_context, type);
37388 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
37390 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
37393 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
37395 gc_heap* hp = (gc_heap*)gc_context;
37396 hp->walk_finalize_queue (fn);
37399 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
37401 #ifdef MULTIPLE_HEAPS
37402 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37404 gc_heap* hp = gc_heap::g_heaps [hn];
37405 hp->finalize_queue->GcScanRoots(fn, hn, sc);
37408 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
37409 #endif //MULTIPLE_HEAPS
37412 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37414 UNREFERENCED_PARAMETER(gen_number);
37415 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
37418 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37420 UNREFERENCED_PARAMETER(gen_number);
37421 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
37424 // Go through and touch (read) each page straddled by a memory block.
37425 void TouchPages(void * pStart, size_t cb)
37427 const uint32_t pagesize = OS_PAGE_SIZE;
37428 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
37431 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
37432 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
37436 a = VolatileLoad(p);
37437 //printf("Touching page %lxh\n", (uint32_t)p);
37443 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
37444 // This code is designed to catch the failure to update the write barrier
37445 // The way it works is to copy the whole heap right after every GC. The write
37446 // barrier code has been modified so that it updates the shadow as well as the
37447 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
37448 // that were updated in the real heap, but not the shadow. A mismatch indicates
37449 // an error. The offending code can be found by breaking after the correct GC,
37450 // and then placing a data breakpoint on the Heap location that was updated without
37451 // going through the write barrier.
37453 // Called at process shutdown
37454 void deleteGCShadow()
37456 if (g_GCShadow != 0)
37457 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
37462 // Called at startup and right after a GC, get a snapshot of the GC Heap
37463 void initGCShadow()
37465 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
37468 size_t len = g_gc_highest_address - g_gc_lowest_address;
37469 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
37472 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
37473 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
37475 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
37476 // If after the assert we decide to allow the program to continue
37477 // running we need to be in a state that will not trigger any
37478 // additional AVs while we fail to allocate a shadow segment, i.e.
37479 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
37484 g_GCShadowEnd += len;
37487 // save the value of g_gc_lowest_address at this time. If this value changes before
37488 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
37489 // large object segment most probably), and the whole shadow segment is inconsistent.
37490 g_shadow_lowest_address = g_gc_lowest_address;
37492 //****** Copy the whole GC heap ******
37494 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
37495 // can produce a NULL result. This is because the initialization has not completed.
37497 generation* gen = gc_heap::generation_of (max_generation);
37498 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37500 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
37501 BOOL small_object_segments = TRUE;
37506 if (small_object_segments)
37508 small_object_segments = FALSE;
37509 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
37515 // Copy the segment
37516 uint8_t* start = heap_segment_mem(seg);
37517 uint8_t* end = heap_segment_allocated (seg);
37518 memcpy(start + delta, start, end - start);
37519 seg = heap_segment_next_rw (seg);
37523 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
37525 // test to see if 'ptr' was only updated via the write barrier.
37526 inline void testGCShadow(Object** ptr)
37528 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
37529 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
37532 // If you get this assertion, someone updated a GC pointer in the heap without
37533 // using the write barrier. To find out who, check the value of
37534 // dd_collection_count (dynamic_data_of (0)). Also
37535 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
37536 // Then put a data breakpoint for the value of 'ptr' Then check every write
37537 // to pointer between the two GCs. The last one is not using the write barrier.
37539 // If the memory of interest does not exist at system startup,
37540 // you need to set the data breakpoint right after the memory gets committed
37541 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
37542 // in the memory window. run until the memory gets mapped. Then you can set
37545 // Note a recent change, we've identified race conditions when updating the gc shadow.
37546 // Throughout the runtime, code will update an address in the gc heap, then erect the
37547 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
37548 // from multiple threads, you can hit this assert even though all involved are using the
37549 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
37550 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
37551 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
37552 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
37553 // TODO: erroneous asserts in here.
37555 if(*shadow!=INVALIDGCVALUE)
37557 #ifdef FEATURE_BASICFREEZE
37558 // Write barriers for stores of references to frozen objects may be optimized away.
37559 if (!gc_heap::frozen_object_p(*ptr))
37560 #endif // FEATURE_BASICFREEZE
37562 _ASSERTE(!"Pointer updated without using write barrier");
37568 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
37574 void testGCShadowHelper (uint8_t* x)
37576 size_t s = size (x);
37577 if (contain_pointers (x))
37579 go_through_object_nostart (method_table(x), x, s, oo,
37580 { testGCShadow((Object**) oo); });
37584 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
37585 void checkGCWriteBarrier()
37587 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
37588 // and the GC shadow segment did not track that change!
37589 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
37591 // No shadow stack, nothing to check.
37596 generation* gen = gc_heap::generation_of (max_generation);
37597 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37599 PREFIX_ASSUME(seg != NULL);
37603 uint8_t* x = heap_segment_mem(seg);
37604 while (x < heap_segment_allocated (seg))
37606 size_t s = size (x);
37607 testGCShadowHelper (x);
37610 seg = heap_segment_next_rw (seg);
37615 // go through large object heap
37616 int alignment = get_alignment_constant(FALSE);
37617 generation* gen = gc_heap::generation_of (max_generation+1);
37618 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37620 PREFIX_ASSUME(seg != NULL);
37624 uint8_t* x = heap_segment_mem(seg);
37625 while (x < heap_segment_allocated (seg))
37627 size_t s = size (x);
37628 testGCShadowHelper (x);
37629 x = x + Align (s, alignment);
37631 seg = heap_segment_next_rw (seg);
37635 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
37637 #endif // !DACCESS_COMPILE
37639 #ifdef FEATURE_BASICFREEZE
37640 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
37642 #ifdef DACCESS_COMPILE
37643 UNREFERENCED_PARAMETER(seg);
37644 UNREFERENCED_PARAMETER(pvContext);
37645 UNREFERENCED_PARAMETER(pfnMethodTable);
37646 UNREFERENCED_PARAMETER(pfnObjRef);
37648 uint8_t *o = heap_segment_mem(seg);
37650 // small heap alignment constant
37651 int alignment = get_alignment_constant(TRUE);
37653 while (o < heap_segment_allocated(seg))
37655 pfnMethodTable(pvContext, o);
37657 if (contain_pointers (o))
37659 go_through_object_nostart (method_table (o), o, size(o), oo,
37662 pfnObjRef(pvContext, oo);
37667 o += Align(size(o), alignment);
37669 #endif //!DACCESS_COMPILE
37671 #endif // FEATURE_BASICFREEZE
37673 #ifndef DACCESS_COMPILE
37674 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
37676 #ifdef BACKGROUND_GC
37677 if (recursive_gc_sync::background_running_p())
37679 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
37680 if (dwRet == WAIT_OBJECT_0)
37682 else if (dwRet == WAIT_TIMEOUT)
37683 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
37685 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
37686 // as there are too many layers in between. The best we can do is to return E_FAIL;
37692 #endif // !DACCESS_COMPILE
37694 void GCHeap::TemporaryEnableConcurrentGC()
37696 #ifdef BACKGROUND_GC
37697 gc_heap::temp_disable_concurrent_p = false;
37698 #endif //BACKGROUND_GC
37701 void GCHeap::TemporaryDisableConcurrentGC()
37703 #ifdef BACKGROUND_GC
37704 gc_heap::temp_disable_concurrent_p = true;
37705 #endif //BACKGROUND_GC
37708 bool GCHeap::IsConcurrentGCEnabled()
37710 #ifdef BACKGROUND_GC
37711 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
37714 #endif //BACKGROUND_GC
37717 void GCHeap::SetFinalizeRunOnShutdown(bool value)
37719 g_fFinalizerRunOnShutDown = value;
37722 void PopulateDacVars(GcDacVars *gcDacVars)
37724 #ifndef DACCESS_COMPILE
37725 assert(gcDacVars != nullptr);
37727 gcDacVars->major_version_number = 1;
37728 gcDacVars->minor_version_number = 0;
37729 gcDacVars->built_with_svr = &g_built_with_svr_gc;
37730 gcDacVars->build_variant = &g_build_variant;
37731 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
37732 gcDacVars->generation_size = sizeof(generation);
37733 gcDacVars->max_gen = &g_max_generation;
37734 #ifndef MULTIPLE_HEAPS
37735 gcDacVars->mark_array = &gc_heap::mark_array;
37736 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
37737 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
37738 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
37739 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
37740 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
37741 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
37742 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
37743 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
37744 gcDacVars->oom_info = &gc_heap::oom_info;
37745 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
37746 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
37747 #ifdef GC_CONFIG_DRIVEN
37748 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
37749 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
37750 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
37751 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
37752 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
37753 #endif // GC_CONFIG_DRIVEN
37754 #ifdef HEAP_ANALYZE
37755 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
37756 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
37757 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
37758 #endif // HEAP_ANALYZE
37760 gcDacVars->n_heaps = &gc_heap::n_heaps;
37761 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
37762 #endif // MULTIPLE_HEAPS
37763 #endif // DACCESS_COMPILE